logs

16s ago2026-07-27 02:17:11 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:11 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:17:11.793Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

16s ago2026-07-27 02:17:11 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:11 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:17:11.702Z",
  "version": "0.1.0"
}

16s ago2026-07-27 02:17:11 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:11 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 168.0,
  "elapsedTimeMs": 169.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25355600",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117731299}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:11.473Z",
  "transactionId": 12246572.0,
  "waiting": 0.0
}

16s ago2026-07-27 02:17:11 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:11 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 62.0,
  "elapsedTimeMs": 63.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25355584",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:11.295Z",
  "transactionId": 12246571.0,
  "waiting": 0.0
}

17s ago2026-07-27 02:17:10 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:17:10.988Z",
  "version": "0.1.0"
}

18s ago2026-07-27 02:17:09 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:09 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:17:09.985Z",
  "version": "0.1.0"
}

18s ago2026-07-27 02:17:09 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:17:09 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:17:09.985Z",
  "version": "0.1.0"
}

18s ago2026-07-27 02:17:09 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 274.0,
  "elapsedTimeMs": 274.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25355318",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117729702}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:09.981Z",
  "transactionId": 12246538.0,
  "waiting": 0.0
}

18s ago2026-07-27 02:17:09 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:17:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 115.0,
  "elapsedTimeMs": 115.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25355287",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118029577}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:17:09.698Z",
  "transactionId": 12246523.0,
  "waiting": 0.0
}

31s ago2026-07-27 02:16:56 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:56 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:16:56.220Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

31s ago2026-07-27 02:16:56 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:56 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:56.135Z",
  "version": "0.1.0"
}

32s ago2026-07-27 02:16:55 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:55 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 204.0,
  "elapsedTimeMs": 204.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25350947",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117715728}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:55.939Z",
  "transactionId": 12244916.0,
  "waiting": 0.0
}

32s ago2026-07-27 02:16:55 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:55 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 65.0,
  "elapsedTimeMs": 65.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25350932",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:55.725Z",
  "transactionId": 12244915.0,
  "waiting": 0.0
}

36s ago2026-07-27 02:16:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:51 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (r:TmfEvaluationRun {status: $running})
    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
    ORDER BY coalesce(r.updatedat, r.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 800.0,
  "cpu": 0.0,
  "elapsedTimeMs": 0.0,
  "executionPlanCacheKeyHash": "95899445",
  "id": "25350212",
  "pageFaults": 0.0,
  "pageHits": 21.0,
  "planning": 0.0,
  "query": "\n    MATCH (r:TmfEvaluationRun {status: $running})\n    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n    ORDER BY coalesce(r.updatedat, r.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785118311892}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:51.898Z",
  "transactionId": 12244911.0,
  "waiting": 0.0
}

37s ago2026-07-27 02:16:50 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:50.364Z",
  "version": "0.1.0"
}

38s ago2026-07-27 02:16:49 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:49.359Z",
  "version": "0.1.0"
}

38s ago2026-07-27 02:16:49 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:49 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:49.359Z",
  "version": "0.1.0"
}

38s ago2026-07-27 02:16:49 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 194.0,
  "elapsedTimeMs": 194.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25349675",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117709155}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:49.354Z",
  "transactionId": 12244908.0,
  "waiting": 0.0
}

38s ago2026-07-27 02:16:49 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 100.0,
  "elapsedTimeMs": 100.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25349648",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785118009046}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:49.151Z",
  "transactionId": 12244907.0,
  "waiting": 0.0
}

47s ago2026-07-27 02:16:40 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:40 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:16:40.649Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

47s ago2026-07-27 02:16:40 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:40 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:40.576Z",
  "version": "0.1.0"
}

47s ago2026-07-27 02:16:40 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:40 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 173.0,
  "elapsedTimeMs": 173.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25347845",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117700158}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:40.336Z",
  "transactionId": 12244896.0,
  "waiting": 0.0
}

47s ago2026-07-27 02:16:40 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:40 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 65.0,
  "elapsedTimeMs": 65.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25347826",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:40.154Z",
  "transactionId": 12244895.0,
  "waiting": 0.0
}

57s ago2026-07-27 02:16:30 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:30.819Z",
  "version": "0.1.0"
}

58s ago2026-07-27 02:16:29 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:29.815Z",
  "version": "0.1.0"
}

58s ago2026-07-27 02:16:29 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:29 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:29.815Z",
  "version": "0.1.0"
}

58s ago2026-07-27 02:16:29 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 179.0,
  "elapsedTimeMs": 179.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25346379",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117689627}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:29.811Z",
  "transactionId": 12244889.0,
  "waiting": 0.0
}

58s ago2026-07-27 02:16:29 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:29 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 101.0,
  "elapsedTimeMs": 101.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25346373",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117989516}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:29.622Z",
  "transactionId": 12244887.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:16:25 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:25 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:16:25.078Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

1m ago2026-07-27 02:16:25 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:25 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:25.008Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:16:24 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:24 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 169.0,
  "elapsedTimeMs": 169.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25346028",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117684590}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:24.764Z",
  "transactionId": 12244884.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:16:24 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:24 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 79.0,
  "elapsedTimeMs": 79.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25346023",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:24.581Z",
  "transactionId": 12244883.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:16:11 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:11 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:11.309Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:16:10 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:16:10.307Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:16:10 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:10.307Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:16:10 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 210.0,
  "elapsedTimeMs": 210.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25344695",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117670088}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:10.303Z",
  "transactionId": 12244868.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:16:10 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 92.0,
  "elapsedTimeMs": 92.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25344687",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117969987}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:10.084Z",
  "transactionId": 12244867.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:16:09 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:09 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:16:09.491Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

1m ago2026-07-27 02:16:09 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:16:09 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:16:09.418Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:16:09 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 263.0,
  "elapsedTimeMs": 263.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25344539",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117668945}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:09.213Z",
  "transactionId": 12244827.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:16:08 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:16:08 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 67.0,
  "elapsedTimeMs": 67.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25344522",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:16:08.941Z",
  "transactionId": 12244817.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:15:53 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:53 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:15:53.862Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

1m ago2026-07-27 02:15:53 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:53 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:53.795Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:15:53 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:53 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 595.0,
  "elapsedTimeMs": 595.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25340824",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 1.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117653001}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:53.602Z",
  "transactionId": 12243211.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:15:52 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:52 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 285.0,
  "elapsedTimeMs": 286.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25340782",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 1.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:52.996Z",
  "transactionId": 12243210.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:15:51 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:51 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:51.754Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:15:50 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:50.749Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:15:50 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:50.749Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:15:50 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 176.0,
  "elapsedTimeMs": 176.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25340457",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117650565}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:50.745Z",
  "transactionId": 12243208.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:15:50 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 97.0,
  "elapsedTimeMs": 97.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25340442",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117950459}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:50.561Z",
  "transactionId": 12243207.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:15:37 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:37 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:15:37.700Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

1m ago2026-07-27 02:15:37 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:37 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:37.625Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:15:37 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:37 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 207.0,
  "elapsedTimeMs": 207.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25338609",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117637192}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:37.404Z",
  "transactionId": 12243194.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:15:37 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:37 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 71.0,
  "elapsedTimeMs": 71.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25338590",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:37.188Z",
  "transactionId": 12243193.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:15:31 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:31 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:31.216Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:15:30 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:30.210Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:15:30 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:30.210Z",
  "version": "0.1.0"
}

1m ago2026-07-27 02:15:30 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 167.0,
  "elapsedTimeMs": 167.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25337476",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117630033}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:30.206Z",
  "transactionId": 12243188.0,
  "waiting": 0.0
}

1m ago2026-07-27 02:15:30 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 93.0,
  "elapsedTimeMs": 94.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25337472",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117929928}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:30.028Z",
  "transactionId": 12243187.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:15:22 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:22 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:15:22.107Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

2m ago2026-07-27 02:15:22 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:22 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:22.032Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:15:21 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:21 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 168.0,
  "elapsedTimeMs": 169.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25336919",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117621679}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:21.852Z",
  "transactionId": 12243180.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:15:21 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:21 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 67.0,
  "elapsedTimeMs": 67.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25336910",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:21.674Z",
  "transactionId": 12243179.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:15:11 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:11 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:11.673Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:15:10 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:15:10.671Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:15:10 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:10.671Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:15:10 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 166.0,
  "elapsedTimeMs": 166.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25336009",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117610496}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:10.667Z",
  "transactionId": 12243168.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:15:10 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 89.0,
  "elapsedTimeMs": 89.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25336005",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117910398}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:10.493Z",
  "transactionId": 12243167.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:15:06 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:06 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:15:06.592Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

2m ago2026-07-27 02:15:06 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:15:06 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:15:06.511Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:15:06 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:06 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 190.0,
  "elapsedTimeMs": 190.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25335096",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117606137}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:06.332Z",
  "transactionId": 12242764.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:15:06 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:15:06 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 70.0,
  "elapsedTimeMs": 71.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25335066",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:15:06.134Z",
  "transactionId": 12242753.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:14:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:51 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (r:TmfEvaluationRun {status: $running})
    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
    ORDER BY coalesce(r.updatedat, r.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 800.0,
  "cpu": 0.0,
  "elapsedTimeMs": 0.0,
  "executionPlanCacheKeyHash": "95899445",
  "id": "25332033",
  "pageFaults": 0.0,
  "pageHits": 21.0,
  "planning": 0.0,
  "query": "\n    MATCH (r:TmfEvaluationRun {status: $running})\n    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n    ORDER BY coalesce(r.updatedat, r.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785118191882}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:51.888Z",
  "transactionId": 12241512.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:14:51 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:51 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:14:51.168Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:14:51 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:51 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:14:51.020Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

2m ago2026-07-27 02:14:50 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:50 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:14:50.945Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:14:50 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 170.0,
  "elapsedTimeMs": 171.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25331845",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117590562}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:50.737Z",
  "transactionId": 12241510.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:14:50 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 61.0,
  "elapsedTimeMs": 61.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25331840",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:50.558Z",
  "transactionId": 12241509.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:14:50 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:14:50.166Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:14:50 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:14:50.166Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:14:50 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 182.0,
  "elapsedTimeMs": 182.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25331782",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117589975}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:50.162Z",
  "transactionId": 12241508.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:14:49 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:49 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 99.0,
  "elapsedTimeMs": 99.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25331767",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117889867}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:49.971Z",
  "transactionId": 12241507.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:14:35 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:35 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:14:35.487Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

2m ago2026-07-27 02:14:35 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:35 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:14:35.405Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:14:35 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:35 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 167.0,
  "elapsedTimeMs": 167.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25330029",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117574996}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:35.168Z",
  "transactionId": 12241493.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:14:34 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:34 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 63.0,
  "elapsedTimeMs": 63.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25330026",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:34.992Z",
  "transactionId": 12241492.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:14:31 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:31 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:14:31.630Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:14:30 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:14:30.626Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:14:30 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:14:30.626Z",
  "version": "0.1.0"
}

2m ago2026-07-27 02:14:30 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 168.0,
  "elapsedTimeMs": 168.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25329709",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117570448}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:30.621Z",
  "transactionId": 12241483.0,
  "waiting": 0.0
}

2m ago2026-07-27 02:14:30 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 98.0,
  "elapsedTimeMs": 98.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25329701",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117870339}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:30.443Z",
  "transactionId": 12241482.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:14:19 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:19 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:14:19.919Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

3m ago2026-07-27 02:14:19 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:19 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:14:19.848Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:14:19 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:19 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 169.0,
  "elapsedTimeMs": 170.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25329083",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117559473}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:19.647Z",
  "transactionId": 12241473.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:14:19 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:19 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 63.0,
  "elapsedTimeMs": 63.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25329078",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:19.469Z",
  "transactionId": 12241472.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:14:11 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:11 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:14:11.102Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:14:10 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:14:10.097Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:14:10 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:14:10.097Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:14:10 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 180.0,
  "elapsedTimeMs": 180.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25328307",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117549908}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:10.093Z",
  "transactionId": 12241463.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:14:09 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 88.0,
  "elapsedTimeMs": 88.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25328296",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117849808}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:09.903Z",
  "transactionId": 12241462.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:14:04 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:04 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:14:04.395Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

3m ago2026-07-27 02:14:04 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:14:04 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:14:04.314Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:14:04 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:04 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 193.0,
  "elapsedTimeMs": 193.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25326283",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117543895}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:04.092Z",
  "transactionId": 12240814.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:14:03 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:14:03 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 78.0,
  "elapsedTimeMs": 78.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25326245",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:14:03.891Z",
  "transactionId": 12240802.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:13:51 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:51 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:13:51.554Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:13:50 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:13:50.552Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:13:50 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:13:50.551Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:13:50 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 167.0,
  "elapsedTimeMs": 167.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25322998",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117530376}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:50.548Z",
  "transactionId": 12239805.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:13:50 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 89.0,
  "elapsedTimeMs": 89.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25322988",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117830277}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:50.371Z",
  "transactionId": 12239804.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:13:48 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:48 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:13:48.795Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

3m ago2026-07-27 02:13:48 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:48 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:13:48.723Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:13:48 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:48 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 177.0,
  "elapsedTimeMs": 177.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25322800",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117528362}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:48.549Z",
  "transactionId": 12239799.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:13:48 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:48 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 63.0,
  "elapsedTimeMs": 64.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25322789",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:48.358Z",
  "transactionId": 12239797.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:13:33 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:33 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:13:33.283Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

3m ago2026-07-27 02:13:33 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:33 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:13:33.210Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:13:33 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:33 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 167.0,
  "elapsedTimeMs": 167.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25321086",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117512855}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:33.027Z",
  "transactionId": 12239786.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:13:32 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:32 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 62.0,
  "elapsedTimeMs": 62.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25321082",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:32.851Z",
  "transactionId": 12239785.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:13:32 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:32 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:13:32.053Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:13:31 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:31 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:13:31.049Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:13:31 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:31 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:13:31.048Z",
  "version": "0.1.0"
}

3m ago2026-07-27 02:13:31 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:31 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 181.0,
  "elapsedTimeMs": 181.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25320820",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117510859}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:31.044Z",
  "transactionId": 12239784.0,
  "waiting": 0.0
}

3m ago2026-07-27 02:13:30 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 101.0,
  "elapsedTimeMs": 101.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25320808",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117810748}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:30.855Z",
  "transactionId": 12239783.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:13:17 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:17 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:13:17.781Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

4m ago2026-07-27 02:13:17 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:17 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:13:17.700Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:13:17 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:17 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 170.0,
  "elapsedTimeMs": 170.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25319438",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117497330}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:17.504Z",
  "transactionId": 12239769.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:13:17 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:17 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 67.0,
  "elapsedTimeMs": 67.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25319421",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:17.326Z",
  "transactionId": 12239768.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:13:12 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:12 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:13:12.507Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:13:11 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:11 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:13:11.502Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:13:11 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:11 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:13:11.502Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:13:11 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:11 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 174.0,
  "elapsedTimeMs": 174.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25318783",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117491318}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:11.498Z",
  "transactionId": 12239764.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:13:11 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:11 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 88.0,
  "elapsedTimeMs": 88.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25318775",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117791220}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:11.314Z",
  "transactionId": 12239763.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:13:02 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:02 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:13:02.247Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

4m ago2026-07-27 02:13:02 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:13:02 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:13:02.178Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:13:01 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:01 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 192.0,
  "elapsedTimeMs": 192.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25316183",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117481793}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:01.989Z",
  "transactionId": 12238789.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:13:01 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:13:01 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 68.0,
  "elapsedTimeMs": 69.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25316155",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:13:01.788Z",
  "transactionId": 12238778.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:12:52 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:52 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:12:52.030Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:12:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:51 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (r:TmfEvaluationRun {status: $running})
    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
    ORDER BY coalesce(r.updatedat, r.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 800.0,
  "cpu": 0.0,
  "elapsedTimeMs": 0.0,
  "executionPlanCacheKeyHash": "95899445",
  "id": "25314270",
  "pageFaults": 0.0,
  "pageHits": 21.0,
  "planning": 0.0,
  "query": "\n    MATCH (r:TmfEvaluationRun {status: $running})\n    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n    ORDER BY coalesce(r.updatedat, r.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785118071871}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:51.877Z",
  "transactionId": 12238107.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:12:50 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:12:50.996Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:12:50 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:12:50.996Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:12:50 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 182.0,
  "elapsedTimeMs": 183.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25314152",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117470804}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:50.991Z",
  "transactionId": 12238106.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:12:50 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 106.0,
  "elapsedTimeMs": 106.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25314139",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117770688}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:50.800Z",
  "transactionId": 12238105.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:12:46 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:46 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:12:46.708Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

4m ago2026-07-27 02:12:46 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:46 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:12:46.602Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:12:46 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:46 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 163.0,
  "elapsedTimeMs": 163.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25313885",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117466236}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:46.410Z",
  "transactionId": 12238098.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:12:46 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:46 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 63.0,
  "elapsedTimeMs": 63.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25313879",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:37236\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:46.231Z",
  "transactionId": 12238097.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:12:31 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:31 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:12:31.583Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:12:31 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:31 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:12:31.156Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

4m ago2026-07-27 02:12:31 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:31 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:12:31.088Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:12:30 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 218.0,
  "elapsedTimeMs": 218.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25312136",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117450716}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:30.946Z",
  "transactionId": 12238083.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:12:30 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 78.0,
  "elapsedTimeMs": 78.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25312107",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-python/6.2.0 Python/3.12.13-final-0 (linux)\t\tclient/10.129.3.15:42034\tserver/10.173.0.8:7687\u003e",
  "timestamp": "2026-07-27T02:12:30.714Z",
  "transactionId": 12238082.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:12:30 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:12:30.581Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:12:30 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:12:30.580Z",
  "version": "0.1.0"
}

4m ago2026-07-27 02:12:30 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 284.0,
  "elapsedTimeMs": 285.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25312030",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117450287}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:30.577Z",
  "transactionId": 12238080.0,
  "waiting": 0.0
}

4m ago2026-07-27 02:12:30 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 117.0,
  "elapsedTimeMs": 117.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25312006",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117750158}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:30.282Z",
  "transactionId": 12238079.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:12:15 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:15 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:12:15.551Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

5m ago2026-07-27 02:12:15 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:15 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:12:15.478Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:12:15 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:15 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 194.0,
  "elapsedTimeMs": 194.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25308970",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117435043}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:15.242Z",
  "transactionId": 12238065.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:12:15 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:15 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 69.0,
  "elapsedTimeMs": 69.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25308957",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:15.039Z",
  "transactionId": 12238064.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:12:11 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:11 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:12:11.418Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:12:10 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:12:10.413Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:12:10 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:12:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:12:10.413Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:12:10 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 174.0,
  "elapsedTimeMs": 174.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25308355",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117430230}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:10.409Z",
  "transactionId": 12238060.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:12:10 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:12:10 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 92.0,
  "elapsedTimeMs": 92.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25308342",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117730127}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:12:10.226Z",
  "transactionId": 12238059.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:11:59 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:59 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:11:59.962Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:59 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:59 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:11:59.888Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:59 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:59 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 177.0,
  "elapsedTimeMs": 177.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25305917",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117419520}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:59.703Z",
  "transactionId": 12236829.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:11:59 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:59 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 76.0,
  "elapsedTimeMs": 76.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25305895",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:59.516Z",
  "transactionId": 12236818.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:11:51 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:51 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:11:51.908Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:50 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:11:50.904Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:50 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:11:50.904Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:50 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 190.0,
  "elapsedTimeMs": 190.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25304559",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117410706}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:50.900Z",
  "transactionId": 12236401.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:11:50 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:50 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 99.0,
  "elapsedTimeMs": 99.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25304550",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117710598}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:50.702Z",
  "transactionId": 12236400.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:11:44 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:44 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:11:44.429Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:44 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:44 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:11:44.354Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:44 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:44 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 173.0,
  "elapsedTimeMs": 173.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25304149",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117403938}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:44.116Z",
  "transactionId": 12236388.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:11:43 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:43 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 67.0,
  "elapsedTimeMs": 68.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25304138",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:43.928Z",
  "transactionId": 12236387.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:11:31 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:31 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:11:31.355Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:30 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:11:30.352Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:30 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:30 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:11:30.352Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:30 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 179.0,
  "elapsedTimeMs": 179.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25302526",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117390164}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:30.348Z",
  "transactionId": 12236376.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:11:30 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:30 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 89.0,
  "elapsedTimeMs": 89.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25302519",
  "pageFaults": 0.0,
  "pageHits": 89927.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117690065}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:30.160Z",
  "transactionId": 12236375.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:11:28 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:28 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:11:28.848Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:28 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:28 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:11:28.772Z",
  "version": "0.1.0"
}

5m ago2026-07-27 02:11:28 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:28 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 178.0,
  "elapsedTimeMs": 178.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25302120",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117388090}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:28.273Z",
  "transactionId": 12236373.0,
  "waiting": 0.0
}

5m ago2026-07-27 02:11:28 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:28 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 79.0,
  "elapsedTimeMs": 79.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25302101",
  "pageFaults": 0.0,
  "pageHits": 66849.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:28.086Z",
  "transactionId": 12236372.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:11:12 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:12 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:11:12.994Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

6m ago2026-07-27 02:11:12 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:12 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:11:12.921Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:11:12 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:12 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 163.0,
  "elapsedTimeMs": 163.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25300225",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117372584}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:12.752Z",
  "transactionId": 12236357.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:11:12 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:12 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 60.0,
  "elapsedTimeMs": 60.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25300222",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:12.579Z",
  "transactionId": 12236356.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:11:10 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:10 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:11:10.835Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:11:09 UTC DEBUG docproc Pull returned 0 messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:09 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:284

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pull returned 0 messages

raw payload (9 fields)

{
  "caller": "handlers/pubsub_handler.py:284",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Pull returned 0 messages",
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:11:09.831Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:11:09 UTC DEBUG docproc Recovery fill

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:11:09 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:429

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Recovery fill

raw payload (13 fields)

{
  "adopt_list_len": 0.0,
  "adopt_started": 0.0,
  "backlog_list_len": 0.0,
  "backlog_started": 0.0,
  "caller": "handlers/pubsub_handler.py:429",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "message": "Recovery fill",
  "remaining_slots": 10.0,
  "service": "docproc",
  "timestamp": "2026-07-27T02:11:09.831Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:11:09 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,
         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count
    RETURN doc_id, org_alias, version_id, study_id, current_retry_count
    ORDER BY doc_id
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 9610664.0,
  "cpu": 179.0,
  "elapsedTimeMs": 180.0,
  "executionPlanCacheKeyHash": "7644B06C",
  "id": "25300051",
  "pageFaults": 0.0,
  "pageHits": 156448.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d.tildaid AS doc_id, o.alias AS org_alias, d.primaryVersionId AS version_id,\n         d.studyId AS study_id, COALESCE(latest_ai.auto_retry_count, 0) AS current_retry_count\n    RETURN doc_id, org_alias, version_id, study_id, current_retry_count\n    ORDER BY doc_id\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 10, max_retries: 5, threshold_ms: 1785117369642}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:09.827Z",
  "transactionId": 12236354.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:11:09 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:11:09 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    MATCH (d)-[:DOC_TO_ORG]->(o:Organization)
    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,
           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,
           o.tildaid AS org_id, o.alias AS org_alias
    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 736.0,
  "cpu": 98.0,
  "elapsedTimeMs": 98.0,
  "executionPlanCacheKeyHash": "E5FA14C6",
  "id": "25300042",
  "pageFaults": 0.0,
  "pageHits": 89963.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    MATCH (d)-[:DOC_TO_ORG]-\u003e(o:Organization)\n    RETURN d.tildaid AS document_id, d.primaryVersionId AS version_id,\n           ai.tildaid AS ai_run_id, d.studyId AS study_id, ai.run_type AS run_type,\n           o.tildaid AS org_id, o.alias AS org_alias\n    ORDER BY COALESCE(ai.updatedat, ai.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{limit: 30, cutoff_ms: 1785117669534}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:11:09.639Z",
  "transactionId": 12236353.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:10:57 UTC DEBUG docproc Reported pending workload

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:10:57 UTC

severityDEBUG

servicedocproc

sourcejobs/hpa_pending_workload.py:176

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Reported pending workload

raw payload (15 fields)

{
  "backlog_capped": 0.0,
  "caller": "jobs/hpa_pending_workload.py:176",
  "cluster": "master",
  "component": "hpa_pending_workload",
  "environment": "DEVELOPMENT",
  "in_progress_count": 0.0,
  "message": "Reported pending workload",
  "num_replicas": 1.0,
  "recovery_workload": 0.0,
  "service": "docproc",
  "signal": 0.0,
  "timestamp": "2026-07-27T02:10:57.512Z",
  "total_workload": 0.0,
  "unacked": 0.0,
  "version": "0.1.0"
}

6m ago2026-07-27 02:10:57 UTC DEBUG docproc Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:10:57 UTC

severityDEBUG

servicedocproc

sourcejobs/cloud_monitoring_metrics.py:296

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0

raw payload (7 fields)

{
  "caller": "jobs/cloud_monitoring_metrics.py:296",
  "cluster": "master",
  "environment": "DEVELOPMENT",
  "message": "Pub/Sub num_unacked result: project_id=tilda-trial-automation subscription_id_in_filter=tilda-document-master-docproc-sub num_series=1 total=0",
  "service": "docproc",
  "timestamp": "2026-07-27T02:10:57.436Z",
  "version": "0.1.0"
}

6m ago2026-07-27 02:10:57 UTC neo4j docproc MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder) WHERE f.name IN ['Staging', 'Pre-Staging'] ...

filter by: service=docproc severity≥INFO

time2026-07-27 02:10:57 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (o:Organization{orgType:"CRO"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)
        WHERE f.name IN ['Staging', 'Pre-Staging']
        AND d.delpending = false AND d.aiEnabled = true AND d.studyId <> ""
        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId <> ""
    OPTIONAL MATCH (d)<-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)
    WITH o, d, ai
    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC
    WITH o, d, collect(ai)[0] AS latest_ai
    WHERE (
        latest_ai IS NULL
        OR latest_ai.status = "aborted"
        OR (
            latest_ai.status = "failed"
            AND COALESCE(latest_ai.auto_retry_count, 0) < $max_retries
            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) < $threshold_ms
        )
    )
    WITH d
    RETURN count(d) AS total

raw payload (15 fields)

{
  "allocatedBytes": 7666360.0,
  "cpu": 190.0,
  "elapsedTimeMs": 190.0,
  "executionPlanCacheKeyHash": "590286C5",
  "id": "25296921",
  "pageFaults": 0.0,
  "pageHits": 156362.0,
  "planning": 0.0,
  "query": "\n    MATCH (o:Organization{orgType:\"CRO\"})-[:DOC_TO_ORG]-(d:Document)-[:INCLUDES]-(f:Folder)\n        WHERE f.name IN [\u0027Staging\u0027, \u0027Pre-Staging\u0027]\n        AND d.delpending = false AND d.aiEnabled = true AND d.studyId \u003c\u003e \"\"\n        AND d.primaryVersionId IS NOT NULL AND d.primaryVersionId \u003c\u003e \"\"\n    OPTIONAL MATCH (d)\u003c-[:AI_RUN_TO_DOCUMENT_LATEST]-(ai:AIRun)\n    WITH o, d, ai\n    ORDER BY d.tildaid, COALESCE(ai.updatedat, ai.createdat) DESC\n    WITH o, d, collect(ai)[0] AS latest_ai\n    WHERE (\n        latest_ai IS NULL\n        OR latest_ai.status = \"aborted\"\n        OR (\n            latest_ai.status = \"failed\"\n            AND COALESCE(latest_ai.auto_retry_count, 0) \u003c $max_retries\n            AND COALESCE(latest_ai.updatedat, latest_ai.createdat) \u003c $threshold_ms\n        )\n    )\n    WITH d\n    RETURN count(d) AS total\n    ",
  "queryParameters": "{max_retries: 5, threshold_ms: 1785117357022}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:10:57.218Z",
  "transactionId": 12234822.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:10:57 UTC neo4j docproc MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document) WHERE d.aiEnabled = true AND ( (ai.status = "in_progress" AND...

filter by: service=docproc severity≥INFO

time2026-07-27 02:10:57 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]->(d:Document)
    WHERE d.aiEnabled = true
    AND (
        (ai.status = "in_progress" AND ($cutoff_ms <= 0 OR COALESCE(ai.updatedat, ai.createdat) < $cutoff_ms))
        OR ai.status = "aborted"
    )
    RETURN count(ai) AS total

raw payload (15 fields)

{
  "allocatedBytes": 488.0,
  "cpu": 68.0,
  "elapsedTimeMs": 69.0,
  "executionPlanCacheKeyHash": "E4907905",
  "id": "25296898",
  "pageFaults": 0.0,
  "pageHits": 66825.0,
  "planning": 0.0,
  "query": "\n    MATCH (ai:AIRun)-[:AI_RUN_TO_DOCUMENT_LATEST]-\u003e(d:Document)\n    WHERE d.aiEnabled = true\n    AND (\n        (ai.status = \"in_progress\" AND ($cutoff_ms \u003c= 0 OR COALESCE(ai.updatedat, ai.createdat) \u003c $cutoff_ms))\n        OR ai.status = \"aborted\"\n    )\n    RETURN count(ai) AS total\n    ",
  "queryParameters": "{cutoff_ms: 0}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:10:57.017Z",
  "transactionId": 12234810.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:10:51 UTC neo4j docproc MATCH (r:TmfEvaluationRun {status: $running}) WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff RETURN...

filter by: service=docproc severity≥INFO

time2026-07-27 02:10:51 UTC

severityINFO

servicedocproc

clustermaster

podserver-2-dev-0

query

MATCH (r:TmfEvaluationRun {status: $running})
    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) < $cutoff
    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion
    ORDER BY coalesce(r.updatedat, r.createdat) ASC
    LIMIT $limit

raw payload (15 fields)

{
  "allocatedBytes": 800.0,
  "cpu": 0.0,
  "elapsedTimeMs": 1.0,
  "executionPlanCacheKeyHash": "95899445",
  "id": "25296368",
  "pageFaults": 0.0,
  "pageHits": 21.0,
  "planning": 0.0,
  "query": "\n    MATCH (r:TmfEvaluationRun {status: $running})\n    WHERE r.deltimestamp = 0 AND coalesce(r.updatedat, r.createdat) \u003c $cutoff\n    RETURN r.tildaid AS tildaid, r.trainingId AS trainingId, r.tmfVersion AS tmfVersion\n    ORDER BY coalesce(r.updatedat, r.createdat) ASC\n    LIMIT $limit\n    ",
  "queryParameters": "{running: \u0027running\u0027, limit: 50, cutoff: 1785117951861}",
  "runtime": "pipelined",
  "source": "bolt-session\tbolt\tneo4j-java/5.28.13-24cd0fecc64de123fd5711190246e8b0f1e16fd0\t\tclient/10.173.2.71:49876\tserver/10.173.0.8:7688\u003e",
  "timestamp": "2026-07-27T02:10:51.866Z",
  "transactionId": 12234698.0,
  "waiting": 0.0
}

6m ago2026-07-27 02:10:50 UTC DEBUG docproc Pulling messages

filter by: service=docproc severity≥DEBUG

time2026-07-27 02:10:50 UTC

severityDEBUG

servicedocproc

sourcehandlers/pubsub_handler.py:240

clustermaster

version0.1.0

poddocproc-7bc76669-ljh8h

message

Pulling messages

raw payload (11 fields)

{
  "caller": "handlers/pubsub_handler.py:240",
  "cluster": "master",
  "component": "pubsub",
  "environment": "DEVELOPMENT",
  "max_messages": 1.0,
  "message": "Pulling messages",
  "pull_timeout_s": 60.0,
  "service": "docproc",
  "subscription_name": "tilda-document-master-docproc-sub",
  "timestamp": "2026-07-27T02:10:50.332Z",
  "version": "0.1.0"
}