Switch to Victoria Metrics

Added auto-restart to docker-compose.yml

Author: dementii-priadko

README.md

@@ -19,7 +19,7 @@ Built for senior DBAs, SREs, and AI systems who need rapid root cause analysis a
 - **Dual-purpose architecture**: Built for both human experts and AI systems requiring structured performance data
 - **Comprehensive query analysis**: Complete `pg_stat_statements` metrics with historical trends and plan variations
 - **Active Session History**: Postgres's answer to Oracle ASH and AWS RDS Performance Insights
-- **Hybrid storage**: Prometheus for metrics, Postgres for query texts — best of both worlds
+- **Hybrid storage**: Victoria Metrics (Prometheus-compatible) for metrics, Postgres for query texts — best of both worlds
 
 > 📖 **Read more**: [postgres_ai monitoring v0.7 announcement](https://postgres.ai/blog/20250722-postgres-ai-v0-7-expert-level-postgresql-monitoring) - detailed technical overview and architecture decisions.
 
@@ -48,7 +48,7 @@ Experience the full monitoring solution: **https://demo.postgres.ai** (login: `d
 ## 🏗️ Architecture
 
 - **Collection**: pgwatch v3 (by Cybertec) for metrics gathering
-- **Storage**: Prometheus for time-series data + Postgres for query texts
+- **Storage**: Victoria Metrics for time-series data + Postgres for query texts
 - **Visualization**: Grafana with expert-designed dashboards
 - **Analysis**: Structured data output for AI system integration
 
@@ -71,7 +71,7 @@ This monitoring solution exposes several ports that **MUST** be properly firewal
 - **Port 3000** (Grafana) - Contains sensitive database metrics and dashboards
 - **Port 58080** (PGWatch Postgres) - Database monitoring interface  
 - **Port 58089** (PGWatch Prometheus) - Database monitoring interface
-- **Port 59090** (Prometheus) - Metrics storage and queries
+- **Port 59090** (Victoria Metrics) - Metrics storage and queries
 - **Port 59091** (PGWatch Prometheus endpoint) - Metrics collection
 - **Port 55000** (Flask API) - Backend API service
 - **Port 55432** (Demo DB) - When using `--demo` option
@@ -98,6 +98,8 @@ grant pg_monitor to postgres_ai_mon;
 grant select on pg_stat_statements to postgres_ai_mon;
 grant select on pg_stat_database to postgres_ai_mon;
 grant select on pg_stat_user_tables to postgres_ai_mon;
+grant select on pg_stat_user_indexes to postgres_ai_mon;
+grant select on pg_index to postgres_ai_mon;
 
 -- Create a public view for pg_statistic access (required for bloat metrics on user schemas)
 create view public.pg_statistic as
@@ -195,7 +197,7 @@ After running quickstart:
 Technical URLs (for advanced users):
 - **Demo DB**: postgresql://postgres:postgres@localhost:55432/target_database
 - **Monitoring**: http://localhost:58080 (PGWatch)
-- **Metrics**: http://localhost:59090 (Prometheus)
+- **Metrics**: http://localhost:59090 (Victoria Metrics)
 
 ## 📖 Help
 

config/pgwatch-prometheus/metrics.yml

@@ -854,34 +854,32 @@ metrics:
   pg_stat_statements:
     sqls:
       11: |
-        WITH ranked_statements as (
+        WITH aggregated_statements as (
           select
-            pg_get_userbyid(userid) as user,
             pg_database.datname,
-            pg_stat_statements.queryid ,
-            pg_stat_statements.plans as plans_total,
-            pg_stat_statements.calls,
-            pg_stat_statements.total_exec_time as exec_time_total,
-            pg_stat_statements.total_plan_time as plan_time_total,
-            pg_stat_statements.rows,
-            (current_setting('block_size')::int * pg_stat_statements.shared_blks_hit) as shared_bytes_hit_total,
-            (current_setting('block_size')::int * pg_stat_statements.shared_blks_read) as shared_bytes_read_total,
-            (current_setting('block_size')::int * pg_stat_statements.shared_blks_dirtied) as shared_bytes_dirtied_total,
-            (current_setting('block_size')::int * pg_stat_statements.shared_blks_written) as shared_bytes_written_total,
-            pg_stat_statements.blk_read_time as block_read_total,
-            pg_stat_statements.blk_write_time as block_write_total,
-            pg_stat_statements.wal_records,
-            pg_stat_statements.wal_fpi,
-            pg_stat_statements.wal_bytes,
-            (current_setting('block_size')::int * pg_stat_statements.temp_blks_read) as temp_bytes_read,
-            (current_setting('block_size')::int * pg_stat_statements.temp_blks_written) as temp_bytes_written,
-            row_number() over (order by total_exec_time desc) as rn
+            pg_stat_statements.queryid,
+            sum(pg_stat_statements.plans) as plans_total,
+            sum(pg_stat_statements.calls) as calls,
+            sum(pg_stat_statements.total_exec_time) as exec_time_total,
+            sum(pg_stat_statements.total_plan_time) as plan_time_total,
+            sum(pg_stat_statements.rows) as rows,
+            sum(current_setting('block_size')::int * pg_stat_statements.shared_blks_hit) as shared_bytes_hit_total,
+            sum(current_setting('block_size')::int * pg_stat_statements.shared_blks_read) as shared_bytes_read_total,
+            sum(current_setting('block_size')::int * pg_stat_statements.shared_blks_dirtied) as shared_bytes_dirtied_total,
+            sum(current_setting('block_size')::int * pg_stat_statements.shared_blks_written) as shared_bytes_written_total,
+            sum(pg_stat_statements.blk_read_time) as block_read_total,
+            sum(pg_stat_statements.blk_write_time) as block_write_total,
+            sum(pg_stat_statements.wal_records) as wal_records,
+            sum(pg_stat_statements.wal_fpi) as wal_fpi,
+            sum(pg_stat_statements.wal_bytes) as wal_bytes,
+            sum(current_setting('block_size')::int * pg_stat_statements.temp_blks_read) as temp_bytes_read,
+            sum(current_setting('block_size')::int * pg_stat_statements.temp_blks_written) as temp_bytes_written
             from pg_stat_statements
             join pg_database
             on pg_database.oid = pg_stat_statements.dbid
+            group by pg_database.datname, pg_stat_statements.queryid
         )
         select
-          ranked_statements.user as tag_user,
           datname as tag_datname,
           queryid as tag_queryid,
           calls::int8 as calls,
@@ -900,36 +898,34 @@ metrics:
           wal_bytes::int8 as wal_bytes,
           temp_bytes_read::int8 as temp_bytes_read,
           temp_bytes_written::int8 as temp_bytes_written
-          from ranked_statements
+          from aggregated_statements
       17: |
-        WITH ranked_statements as (
+        WITH aggregated_statements as (
           select
-            pg_get_userbyid(userid) as user,
             pg_database.datname,
-            pg_stat_statements.queryid ,
-            pg_stat_statements.plans as plans_total,
-            pg_stat_statements.calls,
-            pg_stat_statements.total_exec_time as exec_time_total,
-            pg_stat_statements.total_plan_time as plan_time_total,
-            pg_stat_statements.rows,
-            (current_setting('block_size')::int * pg_stat_statements.shared_blks_hit) as shared_bytes_hit_total,
-            (current_setting('block_size')::int * pg_stat_statements.shared_blks_read) as shared_bytes_read_total,
-            (current_setting('block_size')::int * pg_stat_statements.shared_blks_dirtied) as shared_bytes_dirtied_total,
-            (current_setting('block_size')::int * pg_stat_statements.shared_blks_written) as shared_bytes_written_total,
-            pg_stat_statements.shared_blk_read_time as block_read_total,
-            pg_stat_statements.shared_blk_write_time as block_write_total,
-            pg_stat_statements.wal_records,
-            pg_stat_statements.wal_fpi,
-            pg_stat_statements.wal_bytes,
-            (current_setting('block_size')::int * pg_stat_statements.temp_blks_read) as temp_bytes_read,
-            (current_setting('block_size')::int * pg_stat_statements.temp_blks_written) as temp_bytes_written,
-            row_number() over (order by total_exec_time desc) as rn
+            pg_stat_statements.queryid,
+            sum(pg_stat_statements.plans) as plans_total,
+            sum(pg_stat_statements.calls) as calls,
+            sum(pg_stat_statements.total_exec_time) as exec_time_total,
+            sum(pg_stat_statements.total_plan_time) as plan_time_total,
+            sum(pg_stat_statements.rows) as rows,
+            sum(current_setting('block_size')::int * pg_stat_statements.shared_blks_hit) as shared_bytes_hit_total,
+            sum(current_setting('block_size')::int * pg_stat_statements.shared_blks_read) as shared_bytes_read_total,
+            sum(current_setting('block_size')::int * pg_stat_statements.shared_blks_dirtied) as shared_bytes_dirtied_total,
+            sum(current_setting('block_size')::int * pg_stat_statements.shared_blks_written) as shared_bytes_written_total,
+            sum(pg_stat_statements.shared_blk_read_time) as block_read_total,
+            sum(pg_stat_statements.shared_blk_write_time) as block_write_total,
+            sum(pg_stat_statements.wal_records) as wal_records,
+            sum(pg_stat_statements.wal_fpi) as wal_fpi,
+            sum(pg_stat_statements.wal_bytes) as wal_bytes,
+            sum(current_setting('block_size')::int * pg_stat_statements.temp_blks_read) as temp_bytes_read,
+            sum(current_setting('block_size')::int * pg_stat_statements.temp_blks_written) as temp_bytes_written
             from pg_stat_statements
             join pg_database
             on pg_database.oid = pg_stat_statements.dbid
+            group by pg_database.datname, pg_stat_statements.queryid
         )
         select
-          ranked_statements.user as tag_user,
           datname as tag_datname,
           queryid as tag_queryid,
           calls::int8 as calls,
@@ -948,7 +944,7 @@ metrics:
           wal_bytes::int8 as wal_bytes,
           temp_bytes_read::int8 as temp_bytes_read,
           temp_bytes_written::int8 as temp_bytes_written
-          from ranked_statements
+          from aggregated_statements
     gauges:
       - calls
       - plans_total
@@ -1650,7 +1646,8 @@ metrics:
             pg_relation_size(i.indexrelid) as index_bytes,
             ci.relpages,
             (case when a.amname = 'btree' then true else false end) as idx_is_btree,
-            array_to_string(i.indclass, ', ') as opclasses
+            array_to_string(i.indclass, ', ') as opclasses,
+            si.stats_reset
           from pg_index i
             join pg_class ci on ci.oid = i.indexrelid and ci.relkind = 'i'
             join pg_class cr on cr.oid = i.indrelid and cr.relkind = 'r'
@@ -1679,6 +1676,7 @@ metrics:
             i.relpages,
             idx_is_btree,
             i.opclasses,
+            i.stats_reset,
             (
               select count(1)
               from fk_indexes fi
@@ -1705,7 +1703,8 @@ metrics:
           relpages,
           idx_is_btree,
           opclasses AS tag_opclasses,
-          supports_fk
+          supports_fk,
+          extract(epoch from stats_reset) as stats_reset_epoch
         from index_ratios
         where
           idx_scan = 0
@@ -1858,6 +1857,54 @@ metrics:
     gauges:
       - '*'
 
+  stats_reset:
+    description: >
+      This metric tracks when statistics were last reset for indexes and tables.
+      It provides visibility into the freshness of statistics data, which is essential for understanding
+      the reliability of index and table usage metrics. A recent reset time indicates that usage statistics
+      may not reflect long-term patterns.
+    sqls:
+      11: |
+        with index_stats as (
+          select
+            schemaname as schema_name,
+            indexrelname as index_name,
+            relname as table_name,
+            'index' as stat_type,
+            stats_reset,
+            extract(epoch from stats_reset) as stats_reset_epoch,
+            extract(epoch from now() - stats_reset) as seconds_since_reset
+          from pg_stat_user_indexes
+          where stats_reset is not null
+        ), table_stats as (
+          select
+            schemaname as schema_name,
+            null::text as index_name,
+            relname as table_name,
+            'table' as stat_type,
+            stats_reset,
+            extract(epoch from stats_reset) as stats_reset_epoch,
+            extract(epoch from now() - stats_reset) as seconds_since_reset
+          from pg_stat_user_tables
+          where stats_reset is not null
+        ), combined_stats as (
+          select * from index_stats
+          union all
+          select * from table_stats
+        )
+        select
+          schema_name as tag_schema_name,
+          table_name as tag_table_name,
+          coalesce(index_name, 'table') as tag_index_name,
+          stat_type as tag_stat_type,
+          stats_reset_epoch,
+          seconds_since_reset
+        from combined_stats
+        order by stats_reset desc;
+    gauges:
+      - 'stats_reset_epoch'
+      - 'seconds_since_reset'
+  
   archive_lag:
     description: >
       This metric measures the lag in WAL archive processing.
@@ -2091,6 +2138,8 @@ metrics:
           tidx_blks_hit
         from
           pg_statio_all_tables
+        order by pg_total_relation_size((schemaname || '.' || relname)::regclass) desc
+        limit 5000;
     gauges:
       - '*'
 
@@ -2133,6 +2182,7 @@ presets:
       redundant_indexes: 10800
       unused_indexes: 7200
       rarely_used_indexes: 10800
+      stats_reset: 3600
       archive_lag: 15
       pg_vacuum_progress: 30
   pg_index_pilot:

docker-compose.yml

@@ -54,22 +54,22 @@ services:
       - sink_postgres_data:/var/lib/postgresql/data
       - ./config/sink-postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
 
-  # Prometheus Sink - Storage for metrics in Prometheus format
+  # VictoriaMetrics Sink - Storage for metrics in Prometheus format
   sink-prometheus:
-    image: prom/prometheus:v3.4.2
+    image: victoriametrics/victoria-metrics:v1.105.0
     container_name: sink-prometheus
     ports:
       - "${BIND_HOST:-}59090:9090"
     volumes:
+      - victoria_metrics_data:/victoria-metrics-data
       - ./config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
-      - prometheus_data:/prometheus
     command:
-      - "--config.file=/etc/prometheus/prometheus.yml"
-      - "--storage.tsdb.path=/prometheus"
-      - "--web.console.libraries=/etc/prometheus/console_libraries"
-      - "--web.console.templates=/etc/prometheus/consoles"
-      - "--storage.tsdb.retention.time=200h"
-      - "--web.enable-lifecycle"
+      - "-storageDataPath=/victoria-metrics-data"
+      - "-retentionPeriod=200h"
+      - "-httpListenAddr=:9090"
+      - "-promscrape.config=/etc/prometheus/prometheus.yml"
+      - "-promscrape.config.strictParse=false"
+      - "-promscrape.maxScrapeSize=128000000"
 
   # PGWatch Instance 1 - Monitoring service (Postgres sink)
   pgwatch-postgres:
@@ -236,5 +236,5 @@ services:
 volumes:
   target_db_data:
   sink_postgres_data:
-  prometheus_data:
+  victoria_metrics_data:
   grafana_data: