sample code to generate metrics
#!/usr/bin/env python3
"""
vm_metrics.py - collect system metrics on a Linux VM.
Writes one JSON object per interval (JSON Lines) to a rotating log file, and can
optionally expose the latest values in Prometheus text format over HTTP.
Install dependency:
sudo apt install -y python3-psutil # or: pip3 install psutil
Examples:
python3 vm_metrics.py --interval 10 --stdout
sudo python3 vm_metrics.py --output /var/log/webapp/metrics.log
python3 vm_metrics.py --output ./metrics.log --prometheus-port 9101
"""
import argparse
import json
import logging
import os
import socket
import sys
import threading
import time
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, HTTPServer
from logging.handlers import RotatingFileHandler
try:
import psutil
except ImportError:
sys.exit("psutil is required: sudo apt install -y python3-psutil (or pip3 install psutil)")
HOSTNAME = socket.gethostname()
MB = 1024 * 1024
latest = {} # most recent sample, read by the Prometheus handler
latest_lock = threading.Lock()
stop_event = threading.Event()
def collect(prev):
"""Return (metrics_dict, counters_for_next_rate_calculation)."""
now = time.time()
vm = psutil.virtual_memory()
swap = psutil.swap_memory()
disk = psutil.disk_usage("/")
net = psutil.net_io_counters()
dio = psutil.disk_io_counters()
load1, load5, load15 = os.getloadavg()
counters = {
"ts": now,
"net_sent": net.bytes_sent,
"net_recv": net.bytes_recv,
"disk_read": dio.read_bytes if dio else 0,
"disk_write": dio.write_bytes if dio else 0,
}
# Per-second rates need the previous sample
rates = {"net_sent_bps": 0.0, "net_recv_bps": 0.0,
"disk_read_bps": 0.0, "disk_write_bps": 0.0}
if prev:
dt = max(now - prev["ts"], 0.001)
rates = {
"net_sent_bps": (counters["net_sent"] - prev["net_sent"]) / dt,
"net_recv_bps": (counters["net_recv"] - prev["net_recv"]) / dt,
"disk_read_bps": (counters["disk_read"] - prev["disk_read"]) / dt,
"disk_write_bps": (counters["disk_write"] - prev["disk_write"]) / dt,
}
metrics = {
"timestamp": datetime.now(timezone.utc).isoformat(),
"host": HOSTNAME,
"cpu_percent": psutil.cpu_percent(interval=None),
"cpu_count": psutil.cpu_count(),
"load_1m": round(load1, 2),
"load_5m": round(load5, 2),
"load_15m": round(load15, 2),
"mem_total_mb": round(vm.total / MB, 1),
"mem_used_mb": round((vm.total - vm.available) / MB, 1),
"mem_percent": vm.percent,
"swap_percent": swap.percent,
"disk_root_total_gb": round(disk.total / 1024 ** 3, 2),
"disk_root_used_gb": round(disk.used / 1024 ** 3, 2),
"disk_root_percent": disk.percent,
"process_count": len(psutil.pids()),
"uptime_seconds": int(now - psutil.boot_time()),
}
metrics.update({k: round(v, 1) for k, v in rates.items()})
return metrics, counters
def build_logger(path, stdout):
logger = logging.getLogger("vm_metrics")
logger.setLevel(logging.INFO)
fmt = logging.Formatter("%(message)s") # message only -> pure JSON lines
if path:
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
fh = RotatingFileHandler(path, maxBytes=10 * MB, backupCount=5)
fh.setFormatter(fmt)
logger.addHandler(fh)
if stdout or not path:
sh = logging.StreamHandler(sys.stdout)
sh.setFormatter(fmt)
logger.addHandler(sh)
return logger
class PromHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path != "/metrics":
self.send_response(404)
self.end_headers()
return
with latest_lock:
sample = dict(latest)
lines = []
for key, value in sample.items():
if isinstance(value, (int, float)):
lines.append(f"# TYPE vm_{key} gauge")
lines.append(f'vm_{key}{{host="{HOSTNAME}"}} {value}')
body = ("\n".join(lines) + "\n").encode()
self.send_response(200)
self.send_header("Content-Type", "text/plain; version=0.0.4")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args): # keep the console quiet
pass
def main():
ap = argparse.ArgumentParser(description="Generate VM metrics")
ap.add_argument("--interval", type=float, default=10, help="seconds between samples (default 10)")
ap.add_argument("--output", default=None, help="log file path, e.g. /var/log/webapp/metrics.log")
ap.add_argument("--stdout", action="store_true", help="also print samples to the console")
ap.add_argument("--prometheus-port", type=int, default=0, help="serve /metrics on this port (0 = off)")
ap.add_argument("--count", type=int, default=0, help="stop after N samples (0 = run forever)")
args = ap.parse_args()
try:
logger = build_logger(args.output, args.stdout)
except PermissionError:
sys.exit(f"Cannot write to {args.output}. Run with sudo or choose another --output path.")
if args.prometheus_port:
server = HTTPServer(("0.0.0.0", args.prometheus_port), PromHandler)
threading.Thread(target=server.serve_forever, daemon=True).start()
print(f"Prometheus endpoint: http://0.0.0.0:{args.prometheus_port}/metrics", file=sys.stderr)
psutil.cpu_percent(interval=None) # prime the CPU counter
prev, taken = None, 0
try:
while not stop_event.is_set():
time.sleep(0) if prev else stop_event.wait(1) # short first wait so CPU % is meaningful
metrics, prev = collect(prev)
with latest_lock:
latest.clear()
latest.update(metrics)
logger.info(json.dumps(metrics))
taken += 1
if args.count and taken >= args.count:
break
stop_event.wait(args.interval)
except KeyboardInterrupt:
pass
if __name__ == "__main__":
main()
setup cronjob to run metrics
sudo crontab -e
* * * * * /usr/bin/python3 /opt/vm_metrics.py --count 1 --output /var/log/webapp/metrics.log >> /var/log/webapp/cron.log 2>&1
# verify
sudo crontab -l
grep CRON /var/log/syslog | tail
## Azure Dashboard (Grafana)
* Watch class recording and create Dashboard
- create new dashboard
- import
- ID
- json
## Task
1. create 2 vm run above scrpit and stress cmd
2. verify logs and cpu
3. create dashboard based on metrics and visualizition
