{ "__inputs": [ { "name": "DS_REPLICATOR", "label": "replicator", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": {}, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "11.5.0-pre" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [], "panels": [ { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 79, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The length of time it took to query redo logs from LogMiner. If this query is taking longer than usual, the Oracle database may be taking longer to respond due to heavy load or the SCN interval we are querying is too large.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 1 }, "id": 65, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_query_redo_logs_duration_seconds_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Redo Logs Query Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The number of normal redo logs queried from LogMiner. This should typically increase if the read redo logs time increases. If not, each individual redo log may be larger in size, or the Oracle database is under load.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 6 }, "id": 66, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_query_redo_logs_count_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Redo Logs Query Count", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The length of time it took to query sentinel redo logs from LogMiner. If this value is spiking, this probably means the Oracle instance is under heavy load. If it is gradually increasing, replication may be idle.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 11 }, "id": 62, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_query_sentinel_logs_duration_seconds_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Sentinel Logs Query Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The number of sentinel redo logs queried from LogMiner. If this number is growing larger, replication processing may be stuck, or traffic is idle on the source Oracle database.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 16 }, "id": 63, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_query_sentinel_logs_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Sentinel Logs Query Count", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The length of time it took to update the sentinel table with the most recent Oracle SCN information. This should be a fast update; if it takes longer, the Oracle instance is likely under heavy load.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 21 }, "id": 64, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_update_sentinel_table_duration_seconds_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Update Sentinel Table Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The amount of time it took to find and add load available log files into LogMiner before actually mining redo logs. If this time is growing slower, most likely the Oracle instance is under load, since it should be a quicker query time.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 26 }, "id": 69, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_load_log_files_duration_seconds_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Load Log Files Duration", "type": "timeseries" } ], "title": "LogMiner", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, "id": 78, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The size of the interval from the start SCN to the current Oracle SCN. If this is growing larger than --scnWindowSize, replication is lagging behind and needs to catch up, or replication is just idle. Potential causes could either be that replicator is processing or applying changes slower, or the Oracle source database is taking longer to process LogMiner queries from being under load. Changing the pull interval may have an affect on this size, as typically more frequent pulls mean smaller SCN intervals, but we may still be bottlenecked by query latency.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 2 }, "id": 58, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_scn_interval_size_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "SCN Interval Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The amount of time it took to fully process an SCN interval. If this is getting too large relative to your baseline, it could potentially imply that queries on the Oracle source database are slowing down, the replication loop is blocked, replicator is slowing down in processing, .etc.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 7 }, "id": 73, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_time_per_window_seconds_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "SCN Interval Iteration Time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The total time from when a inflight transaction was created in replicator memory to when it was fully processed and set as ready to be committed/applied on the target database, but not applied yet. This inceases if there are either long-running transactions on the Oracle database, or the frontend of replicator is lagging behind or blocked (ex. because the consumer is yet to read from the channel, slow iterations, .etc).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 12 }, "id": 56, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_inflight_transaction_lifetime_seconds_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Inflight Transaction Lifetime", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "For internal benchmarking only. The amount of lag from the modification of a row on the source, to when it arrives at the replicator frontend to be processed. This can help diagnose if the amount of time it takes for changes to reach replicator is taking too long, which could be caused by replication lagging behind, or the Oracle instance being slow to process and deliver redo logs, or network latency.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 18 }, "id": 61, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_source_lag_from_row_modification_seconds_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Source Row Modification to Replicator Lag", "type": "timeseries" } ], "title": "Throughput", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, "id": 81, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The current number of inflight transactions in memory. Useful to see how many ongoing transactions replicator is keeping track of when replaying from its current replication SCN, as well as for memory usage.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 3 }, "id": 57, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "oraclelogminer_num_inflight_transactions_in_memory", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Number of Current Inflight Transactions", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The average number of mutations within an inflight transaction. Useful to tell how many DMLs are contained in transactions, and for memory usage when combined with other metrics like the number of inflight transactions and mutation size.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 8 }, "id": 68, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_avg_mutations_per_inflight_transaction_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Average Number of Mutations Per Inflight Transaction", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The estimated byte size of a mutation that was just created. This typically increases as the DML statements executed on the source database increase in size. This could be something like larger values for VARCHARs.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 13 }, "id": 67, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_estimated_mutation_byte_size_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Estimated Mutation Byte Size", "type": "timeseries" } ], "title": "Memory Usage", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, "id": 83, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "Number of Oracle source replication checkpoints currently queued up for processing against the staging database. If this is growing too close to the max --checkpointQueueBufferSize, this could indicate that replicator is processing transactions from source to target quicker than it is processing checkpoints for these transactions on the staging database. Try increasing the --checkpointBatchUpsertSize.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 136 }, "id": 85, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_num_async_checkpoints_in_queue_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Number of Asynchronous Checkpoints in Queue", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "Number of checkpoints to be upserted next on the staging CockroachDB. This number may increase from the initial --checkpointBatchUpsertSize value to handle cases where checkpoint producers are speeding up and the async checkpointer consumer needs to increase its rate of checkpoint persistence.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 141 }, "id": 86, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_num_checkpoints_for_batch_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Number of Checkpoints in Batch", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "Amount of time to upsert a checkpoint operation batch into the staging database. If this time is too large relative to your baseline, it could indicate that the staging database is under heavy load, or that the --checkpointBatchUpsertSize is too large, which could be a consequence of the checkpointer trying to catch up with all the transaction checkpoints being produced.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 146 }, "id": 84, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_upsert_checkpoints_duration_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Upsert Checkpoint Batch Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "Amount of time to delete old and unnecessary checkpoints in the staging database. If this time is too large relative to your baseline, it could indicate that the staging database is under heavy load, or that there were some long-running transactions that prevented many checkpoints from being deleted.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 151 }, "id": 87, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_delete_checkpoints_duration_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Delete Old Checkpoints Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "Number of checkpoints deleted that were no longer necessary for successful replicator restart. If this grows larger, it may be due to increased replication transaction throughput, or because there are some long-running transactions preventing other checkpoints from being deleted.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 156 }, "id": 88, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_num_checkpoints_deleted_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Delete Old Checkpoints Count", "type": "timeseries" } ], "title": "Checkpointing", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "id": 82, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The number of start redo logs processed in an SCN window. If this is much lower than the number of end logs, it could be an indicator of longer running transactions to keep track of in replicator, leading to increased processing time and memory usage.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 5 }, "id": 74, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_num_start_logs_per_window_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Number of Start Transaction Logs per SCN Window", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The number of end redo logs processed in an SCN window. If this is much higher than the number of start logs, it could be an indicator of longer running transactions to keep track of in replicator, leading to increased processing time and memory usage.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 10 }, "id": 75, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_num_end_logs_per_window_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Number of End Transaction Logs per SCN Window", "type": "timeseries" } ], "title": "Oracle Workload", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 80, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The amount of time it takes to fully parse a redo log. A larger amount of time means more replicator process CPU is being spent on parsing redo log strings provided by Oracle LogMiner, which may be due to a slower parsing mode being used as a failback from the faster mode, or INSERT/UPDATE statements with many/larger values or many columns, complex expressions, .etc", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 6 }, "id": 76, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le) (rate(oraclelogminer_redo_log_parsing_duration_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Redo Log Parsing Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" }, "description": "The number of SQL parsing failures that occurred in fast parsing mode, so we had to fail back to slow parsing mode. If there are many of these, it could mean that the expressions in the DML statements on the source are more complex in structure, and may need further special handling in replicator if throughput is slow.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 12 }, "id": 77, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.5.0-pre", "targets": [ { "disableTextWrap": false, "editorMode": "builder", "expr": "oraclelogminer_num_antlr_sll_mode_failures", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false, "datasource": { "type": "prometheus", "uid": "${DS_REPLICATOR}" } } ], "title": "Number of Redo Log Parsing Failures in Fast Parsing Mode", "type": "timeseries" } ], "title": "Parsing", "type": "row" } ], "refresh": "auto", "schemaVersion": 40, "tags": [], "templating": { "list": [] }, "time": { "from": "now-5m", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Oracle Source Metrics", "uid": "aeiln2ihybaiob", "version": 80, "weekStart": "" }