logs

9s ago2026-07-27 02:21:05 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:21:05 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:21:05.144Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

9s ago2026-07-27 02:21:05 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:21:05 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:21:05.069Z",
  "version": "0.1.0"
}

23s ago2026-07-27 02:20:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:51 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (r:TmfEvaluationRun {status: $running})
    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
    ORDER BY coalesce(r.updatedat, r.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 800.0,
  "cpu": 0.0,
  "elapsedTimeMs": 1.0,
  "executionPlanCacheKeyHash": "95899445",
  "id": "25378612",
  "pageFaults": 0.0,
  "pageHits": 21.0,
  "planning": 0.0,
  "query": "\n    MATCH (r:TmfEvaluationRun {status: $running})\n    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n    ORDER BY coalesce(r.updatedat, r.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785118551913}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:51.919Z",
  "transactionId": 12251722.0,
  "waiting": 0.0
}

25s ago2026-07-27 02:20:49 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:20:49.689Z",
  "version": "0.1.0"
}

25s ago2026-07-27 02:20:49 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:49 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:20:49.598Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

25s ago2026-07-27 02:20:49 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:49 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:20:49.516Z",
  "version": "0.1.0"
}

25s ago2026-07-27 02:20:49 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 171.0,
  "elapsedTimeMs": 171.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25378560",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117949123}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:49.300Z",
  "transactionId": 12251717.0,
  "waiting": 0.0
}

25s ago2026-07-27 02:20:49 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 62.0,
  "elapsedTimeMs": 62.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25378559",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:49.119Z",
  "transactionId": 12251716.0,
  "waiting": 0.0
}

26s ago2026-07-27 02:20:48 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:48 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:20:48.687Z",
  "version": "0.1.0"
}

26s ago2026-07-27 02:20:48 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:48 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:20:48.687Z",
  "version": "0.1.0"
}

26s ago2026-07-27 02:20:48 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:48 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 165.0,
  "elapsedTimeMs": 165.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25378548",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117948512}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:48.682Z",
  "transactionId": 12251711.0,
  "waiting": 0.0
}

26s ago2026-07-27 02:20:48 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:48 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 86.0,
  "elapsedTimeMs": 87.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25378545",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118248416}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:48.508Z",
  "transactionId": 12251710.0,
  "waiting": 0.0
}

40s ago2026-07-27 02:20:34 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:34 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:20:34.045Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

41s ago2026-07-27 02:20:33 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:33 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:20:33.953Z",
  "version": "0.1.0"
}

41s ago2026-07-27 02:20:33 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:33 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 171.0,
  "elapsedTimeMs": 171.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25377813",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117933584}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:33.760Z",
  "transactionId": 12251698.0,
  "waiting": 0.0
}

41s ago2026-07-27 02:20:33 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:33 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 62.0,
  "elapsedTimeMs": 62.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25377809",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:33.579Z",
  "transactionId": 12251697.0,
  "waiting": 0.0
}

44s ago2026-07-27 02:20:30 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:20:30.161Z",
  "version": "0.1.0"
}

45s ago2026-07-27 02:20:29 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:20:29.156Z",
  "version": "0.1.0"
}

45s ago2026-07-27 02:20:29 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:20:29.156Z",
  "version": "0.1.0"
}

45s ago2026-07-27 02:20:29 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 162.0,
  "elapsedTimeMs": 162.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25377529",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117928983}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:29.151Z",
  "transactionId": 12251694.0,
  "waiting": 0.0
}

46s ago2026-07-27 02:20:28 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:28 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 84.0,
  "elapsedTimeMs": 84.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25377528",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118228887}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:28.977Z",
  "transactionId": 12251693.0,
  "waiting": 0.0
}

56s ago2026-07-27 02:20:18 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:18 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:20:18.509Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

56s ago2026-07-27 02:20:18 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:18 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:20:18.440Z",
  "version": "0.1.0"
}

56s ago2026-07-27 02:20:18 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:18 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 183.0,
  "elapsedTimeMs": 183.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25377396",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117918064}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:18.252Z",
  "transactionId": 12251681.0,
  "waiting": 0.0
}

56s ago2026-07-27 02:20:18 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:18 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 61.0,
  "elapsedTimeMs": 61.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25377395",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:18.060Z",
  "transactionId": 12251680.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:20:10 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:20:10.676Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:20:09 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:09 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:20:09.671Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:20:09 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:09 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:20:09.671Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:20:09 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 168.0,
  "elapsedTimeMs": 168.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25377224",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117909494}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:09.667Z",
  "transactionId": 12251667.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:20:09 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 93.0,
  "elapsedTimeMs": 93.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25377198",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118209357}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:09.490Z",
  "transactionId": 12251653.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:20:02 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:02 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:20:02.987Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

1m ago2026-07-27 02:20:02 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:20:02 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:20:02.899Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:20:02 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:02 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 184.0,
  "elapsedTimeMs": 184.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25375937",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117902516}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:02.711Z",
  "transactionId": 12250807.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:20:02 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:20:02 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 61.0,
  "elapsedTimeMs": 61.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25375923",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:20:02.511Z",
  "transactionId": 12250797.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:19:50 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:19:50.103Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:49 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:19:49.100Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:49 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:19:49.100Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:49 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 167.0,
  "elapsedTimeMs": 167.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25374232",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117888923}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:49.095Z",
  "transactionId": 12250015.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:19:48 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:48 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 90.0,
  "elapsedTimeMs": 90.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25374228",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118188824}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:48.919Z",
  "transactionId": 12250014.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:19:47 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:47 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:19:47.439Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:47 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:47 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:19:47.374Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:47 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:47 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 165.0,
  "elapsedTimeMs": 165.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25374069",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117886972}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:47.143Z",
  "transactionId": 12250009.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:19:46 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:46 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 82.0,
  "elapsedTimeMs": 82.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25374060",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:46.968Z",
  "transactionId": 12250008.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:19:31 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:31 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:19:31.874Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:31 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:31 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:19:31.794Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:31 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:31 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 212.0,
  "elapsedTimeMs": 213.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25372293",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117871384}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:31.605Z",
  "transactionId": 12249997.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:19:31 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:31 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 72.0,
  "elapsedTimeMs": 73.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25372281",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:31.380Z",
  "transactionId": 12249996.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:19:30 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:19:30.599Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:29 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:19:29.596Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:29 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:19:29.596Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:29 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 184.0,
  "elapsedTimeMs": 184.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25371930",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117869402}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:29.591Z",
  "transactionId": 12249993.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:19:29 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 98.0,
  "elapsedTimeMs": 98.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25371918",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118169295}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:29.398Z",
  "transactionId": 12249992.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:19:16 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:16 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:19:16.300Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:16 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:16 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:19:16.232Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:19:16 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:16 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 163.0,
  "elapsedTimeMs": 163.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25370431",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117855847}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:16.015Z",
  "transactionId": 12249980.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:19:15 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:15 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 62.0,
  "elapsedTimeMs": 62.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25370422",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:15.842Z",
  "transactionId": 12249979.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:19:11 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:11 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:19:11.058Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:19:10 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:19:10.054Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:19:10 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:19:10.053Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:19:10 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 179.0,
  "elapsedTimeMs": 179.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25369944",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117849866}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:10.050Z",
  "transactionId": 12249974.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:19:09 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 91.0,
  "elapsedTimeMs": 91.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25369931",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118149766}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:09.862Z",
  "transactionId": 12249973.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:19:00 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:00 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:19:00.770Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

2m ago2026-07-27 02:19:00 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:19:00 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:19:00.697Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:19:00 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:00 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 172.0,
  "elapsedTimeMs": 172.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25367729",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117840306}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:00.483Z",
  "transactionId": 12248841.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:19:00 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:19:00 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 67.0,
  "elapsedTimeMs": 67.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25367711",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:19:00.302Z",
  "transactionId": 12248831.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:18:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:51 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (r:TmfEvaluationRun {status: $running})
    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
    ORDER BY coalesce(r.updatedat, r.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 800.0,
  "cpu": 0.0,
  "elapsedTimeMs": 1.0,
  "executionPlanCacheKeyHash": "95899445",
  "id": "25366444",
  "pageFaults": 0.0,
  "pageHits": 21.0,
  "planning": 0.0,
  "query": "\n    MATCH (r:TmfEvaluationRun {status: $running})\n    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n    ORDER BY coalesce(r.updatedat, r.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785118431903}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:51.909Z",
  "transactionId": 12248318.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:18:50 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:18:50.530Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:18:49 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:18:49.526Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:18:49 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:18:49.526Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:18:49 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 182.0,
  "elapsedTimeMs": 182.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25365913",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117829335}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:49.522Z",
  "transactionId": 12248315.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:18:49 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 94.0,
  "elapsedTimeMs": 94.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25365894",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118129232}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:49.331Z",
  "transactionId": 12248314.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:18:45 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:45 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:18:45.224Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

2m ago2026-07-27 02:18:45 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:45 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:18:45.159Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:18:44 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:44 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 183.0,
  "elapsedTimeMs": 183.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25365463",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117824765}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:44.953Z",
  "transactionId": 12248308.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:18:44 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:44 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 94.0,
  "elapsedTimeMs": 94.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25365447",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:44.761Z",
  "transactionId": 12248307.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:18:30 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:18:30.050Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:18:29 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:29 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:18:29.656Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

2m ago2026-07-27 02:18:29 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:29 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:18:29.584Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:18:29 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 275.0,
  "elapsedTimeMs": 275.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25364390",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117809100}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:29.380Z",
  "transactionId": 12248295.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:18:29 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 95.0,
  "elapsedTimeMs": 96.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25364384",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
  "timestamp": "2026-07-27T02:18:29.098Z",
  "transactionId": 12248294.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:18:29 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:18:29.048Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:18:29 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:18:29.047Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:18:29 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 216.0,
  "elapsedTimeMs": 217.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25364362",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117808822}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:29.043Z",
  "transactionId": 12248293.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:18:28 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:28 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 111.0,
  "elapsedTimeMs": 111.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25364356",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118108701}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:28.818Z",
  "transactionId": 12248292.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:18:13 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:13 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:18:13.994Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

3m ago2026-07-27 02:18:13 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:13 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:18:13.918Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:18:13 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:13 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 171.0,
  "elapsedTimeMs": 171.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25363386",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117793508}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:13.684Z",
  "transactionId": 12248272.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:18:13 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:13 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 60.0,
  "elapsedTimeMs": 60.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25363382",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:13.504Z",
  "transactionId": 12248271.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:18:10 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:18:10.494Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:18:09 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:09 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:18:09.490Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:18:09 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:18:09 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:18:09.489Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:18:09 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 198.0,
  "elapsedTimeMs": 199.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25362974",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117789283}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:09.486Z",
  "transactionId": 12248189.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:18:09 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:18:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 102.0,
  "elapsedTimeMs": 102.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25362937",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118089171}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:18:09.279Z",
  "transactionId": 12248175.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:17:58 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:58 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:17:58.435Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:58 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:58 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:17:58.362Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:58 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:58 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 175.0,
  "elapsedTimeMs": 176.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25360397",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117777981}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:58.161Z",
  "transactionId": 12246826.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:17:57 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:57 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 65.0,
  "elapsedTimeMs": 66.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25360381",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:57.976Z",
  "transactionId": 12246816.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:17:50 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:17:50.416Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:49 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:17:49.413Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:49 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:17:49.412Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:49 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 166.0,
  "elapsedTimeMs": 166.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25359583",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117769237}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:49.408Z",
  "transactionId": 12246610.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:17:49 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 88.0,
  "elapsedTimeMs": 88.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25359576",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118069139}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:49.233Z",
  "transactionId": 12246609.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:17:42 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:42 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:17:42.900Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:42 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:42 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:17:42.824Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:42 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:42 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 165.0,
  "elapsedTimeMs": 165.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25359007",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117762444}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:42.620Z",
  "transactionId": 12246599.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:17:42 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:42 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 76.0,
  "elapsedTimeMs": 76.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25358994",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:42.440Z",
  "transactionId": 12246598.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:17:30 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:17:30.413Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:29 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:17:29.410Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:29 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:17:29.410Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:29 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 173.0,
  "elapsedTimeMs": 174.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25357811",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117749227}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:29.406Z",
  "transactionId": 12246590.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:17:29 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 109.0,
  "elapsedTimeMs": 109.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25357794",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118049108}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:29.223Z",
  "transactionId": 12246589.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:17:27 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:27 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:17:27.352Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:27 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:27 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:17:27.265Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:17:27 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:27 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 165.0,
  "elapsedTimeMs": 165.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25357596",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117746877}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:27.051Z",
  "transactionId": 12246588.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:17:26 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:26 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 67.0,
  "elapsedTimeMs": 67.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25357583",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:26.872Z",
  "transactionId": 12246587.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:17:11 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:11 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:17:11.793Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

4m ago2026-07-27 02:17:11 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:11 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:17:11.702Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:17:11 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:11 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 168.0,
  "elapsedTimeMs": 169.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25355600",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117731299}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:11.473Z",
  "transactionId": 12246572.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:17:11 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:11 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 62.0,
  "elapsedTimeMs": 63.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25355584",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:11.295Z",
  "transactionId": 12246571.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:17:10 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:17:10.988Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:17:09 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:09 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:17:09.985Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:17:09 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:09 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:17:09.985Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:17:09 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 274.0,
  "elapsedTimeMs": 274.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25355318",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117729702}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:09.981Z",
  "transactionId": 12246538.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:17:09 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 115.0,
  "elapsedTimeMs": 115.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25355287",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118029577}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:09.698Z",
  "transactionId": 12246523.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:56 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:56 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:16:56.220Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:56 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:56 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:56.135Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:55 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:55 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 204.0,
  "elapsedTimeMs": 204.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25350947",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117715728}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:55.939Z",
  "transactionId": 12244916.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:55 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:55 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 65.0,
  "elapsedTimeMs": 65.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25350932",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:55.725Z",
  "transactionId": 12244915.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:51 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (r:TmfEvaluationRun {status: $running})
    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
    ORDER BY coalesce(r.updatedat, r.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 800.0,
  "cpu": 0.0,
  "elapsedTimeMs": 0.0,
  "executionPlanCacheKeyHash": "95899445",
  "id": "25350212",
  "pageFaults": 0.0,
  "pageHits": 21.0,
  "planning": 0.0,
  "query": "\n    MATCH (r:TmfEvaluationRun {status: $running})\n    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n    ORDER BY coalesce(r.updatedat, r.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785118311892}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:51.898Z",
  "transactionId": 12244911.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:50 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:50.364Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:49 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:49.359Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:49 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:49.359Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:49 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 194.0,
  "elapsedTimeMs": 194.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25349675",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117709155}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:49.354Z",
  "transactionId": 12244908.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:49 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 100.0,
  "elapsedTimeMs": 100.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25349648",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118009046}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:49.151Z",
  "transactionId": 12244907.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:40 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:40 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:16:40.649Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:40 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:40 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:40.576Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:40 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:40 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 173.0,
  "elapsedTimeMs": 173.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25347845",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117700158}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:40.336Z",
  "transactionId": 12244896.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:40 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:40 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 65.0,
  "elapsedTimeMs": 65.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25347826",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:40.154Z",
  "transactionId": 12244895.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:30 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:30.819Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:29 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:29.815Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:29 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:29.815Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:29 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 179.0,
  "elapsedTimeMs": 179.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25346379",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117689627}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:29.811Z",
  "transactionId": 12244889.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:29 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 101.0,
  "elapsedTimeMs": 101.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25346373",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117989516}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:29.622Z",
  "transactionId": 12244887.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:25 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:25 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:16:25.078Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:25 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:25 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:25.008Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:16:24 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:24 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 169.0,
  "elapsedTimeMs": 169.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25346028",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117684590}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:24.764Z",
  "transactionId": 12244884.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:16:24 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:24 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 79.0,
  "elapsedTimeMs": 79.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25346023",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:24.581Z",
  "transactionId": 12244883.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:16:11 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:11 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:11.309Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:16:10 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:10.307Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:16:10 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:10.307Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:16:10 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 210.0,
  "elapsedTimeMs": 210.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25344695",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117670088}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:10.303Z",
  "transactionId": 12244868.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:16:10 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 92.0,
  "elapsedTimeMs": 92.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25344687",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117969987}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:10.084Z",
  "transactionId": 12244867.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:16:09 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:09 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:16:09.491Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

5m ago2026-07-27 02:16:09 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:09 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:09.418Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:16:09 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 263.0,
  "elapsedTimeMs": 263.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25344539",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117668945}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:09.213Z",
  "transactionId": 12244827.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:16:08 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:08 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 67.0,
  "elapsedTimeMs": 67.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25344522",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:08.941Z",
  "transactionId": 12244817.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:15:53 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:53 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:15:53.862Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:53 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:53 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:53.795Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:53 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:53 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 595.0,
  "elapsedTimeMs": 595.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25340824",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 1.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117653001}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:53.602Z",
  "transactionId": 12243211.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:15:52 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:52 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 285.0,
  "elapsedTimeMs": 286.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25340782",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 1.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:52.996Z",
  "transactionId": 12243210.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:15:51 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:51 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:51.754Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:50 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:50.749Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:50 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:50.749Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:50 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 176.0,
  "elapsedTimeMs": 176.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25340457",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117650565}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:50.745Z",
  "transactionId": 12243208.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:15:50 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 97.0,
  "elapsedTimeMs": 97.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25340442",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117950459}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:50.561Z",
  "transactionId": 12243207.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:15:37 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:37 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:15:37.700Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:37 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:37 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:37.625Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:37 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:37 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 207.0,
  "elapsedTimeMs": 207.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25338609",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117637192}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:37.404Z",
  "transactionId": 12243194.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:15:37 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:37 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 71.0,
  "elapsedTimeMs": 71.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25338590",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:37.188Z",
  "transactionId": 12243193.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:15:31 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:31 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:31.216Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:30 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:30.210Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:30 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:30.210Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:30 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 167.0,
  "elapsedTimeMs": 167.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25337476",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117630033}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:30.206Z",
  "transactionId": 12243188.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:15:30 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 93.0,
  "elapsedTimeMs": 94.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25337472",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117929928}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:30.028Z",
  "transactionId": 12243187.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:15:22 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:22 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:15:22.107Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:22 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:22 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:22.032Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:15:21 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:21 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 168.0,
  "elapsedTimeMs": 169.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25336919",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117621679}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:21.852Z",
  "transactionId": 12243180.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:15:21 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:21 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 67.0,
  "elapsedTimeMs": 67.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25336910",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:21.674Z",
  "transactionId": 12243179.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:15:11 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:11 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:11.673Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:15:10 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:10.671Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:15:10 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:10.671Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:15:10 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 166.0,
  "elapsedTimeMs": 166.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25336009",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117610496}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:10.667Z",
  "transactionId": 12243168.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:15:10 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 89.0,
  "elapsedTimeMs": 89.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25336005",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117910398}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:10.493Z",
  "transactionId": 12243167.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:15:06 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:06 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:15:06.592Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

6m ago2026-07-27 02:15:06 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:06 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:06.511Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:15:06 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:06 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 190.0,
  "elapsedTimeMs": 190.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25335096",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117606137}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:06.332Z",
  "transactionId": 12242764.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:15:06 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:06 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 70.0,
  "elapsedTimeMs": 71.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25335066",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:06.134Z",
  "transactionId": 12242753.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:14:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:51 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (r:TmfEvaluationRun {status: $running})
    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
    ORDER BY coalesce(r.updatedat, r.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 800.0,
  "cpu": 0.0,
  "elapsedTimeMs": 0.0,
  "executionPlanCacheKeyHash": "95899445",
  "id": "25332033",
  "pageFaults": 0.0,
  "pageHits": 21.0,
  "planning": 0.0,
  "query": "\n    MATCH (r:TmfEvaluationRun {status: $running})\n    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n    ORDER BY coalesce(r.updatedat, r.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785118191882}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:51.888Z",
  "transactionId": 12241512.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:14:51 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:51 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:14:51.168Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:14:51 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:51 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:14:51.020Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

6m ago2026-07-27 02:14:50 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:50 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:14:50.945Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:14:50 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 170.0,
  "elapsedTimeMs": 171.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25331845",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117590562}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:50.737Z",
  "transactionId": 12241510.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:14:50 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 61.0,
  "elapsedTimeMs": 61.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25331840",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:50.558Z",
  "transactionId": 12241509.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:14:50 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:14:50.166Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:14:50 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:14:50.166Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:14:50 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 182.0,
  "elapsedTimeMs": 182.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25331782",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117589975}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:50.162Z",
  "transactionId": 12241508.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:14:49 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 99.0,
  "elapsedTimeMs": 99.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25331767",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117889867}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:49.971Z",
  "transactionId": 12241507.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:14:35 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:35 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:14:35.487Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

6m ago2026-07-27 02:14:35 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:35 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:14:35.405Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:14:35 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:35 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 167.0,
  "elapsedTimeMs": 167.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25330029",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117574996}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:35.168Z",
  "transactionId": 12241493.0,
  "waiting": 0.0
}