Beta
Table of Contents
The outline of your notebook will show up here. You can include headings in any text cell by starting a line with # , ## , ### , etc., depending on the desired title hierarchy.
# Start coding here...
import pandas as pd
import numpy as np
from itertools import permutations
from collections import Counter, deque
from functools import reduce
from typing import List
import matplotlib.pyplot as plt
import seaborn as sns
import math
from bisect import bisect_left as bsl
import re
import yamlstrings = """
---
all:
children:
# csi cluster for proxy, monitor, alert, etc..
csi: { hosts: { 10.10.10.10: { csi_seq: 1 } } }
# etcd cluster for ha postgres
etcd:
hosts:
10.10.10.11: { etcd_seq: 1 }
10.10.10.12: { etcd_seq: 2 }
10.10.10.13: { etcd_seq: 3 }
vars: { etcd_cluster: etcd-lucas }
# postgres cluster 'pg-meta' with single primary instance
pg-lucas:
hosts:
10.10.10.11: { pg_seq: 1, pg_role: primary }
10.10.10.12: { pg_seq: 2, pg_role: replica }
vars:
pg_cluster: pg-lucas
node_crontab: [ '00 03 * * * postgres /opt/CrystalDB/bin/pg-backup full' ]
pg_vip_enabled: true
pg_vip_address: 10.10.10.22/24
pg_vip_interface: eth0
vars: # global parameters
version: v2.2.0 # crystaldb version string
admin_ip: 10.10.10.10 # admin node ip address
region: default # upstream mirror region: default,china,europe
...
"""
with open("crystaldb.yml", "w") as file:
file.write(strings)
auto = """
---
#==============================================================#
# File : auto.yml
# Desc : CSI auto generated config for el9,8,7 singleton
# Ctime : 2020-05-22
# Mtime : 2023-03-31
# Docs : https://vonng.github.io/crystaldb/#/CONFIG
# Author : CrystalDB LLC
# License : CrystalDB ELA
#==============================================================#
# this is a simple singleton meta config template, check full details with
# https://github.com/Vonng/crystaldb/blob/master/files/crystaldb/full.yml
all:
children:
# csi cluster for proxy, monitor, alert, etc..
csi: { hosts: { 10.10.10.10: { csi_seq: 1 } } }
# etcd cluster for ha postgres
etcd: { hosts: { 10.10.10.10: { etcd_seq: 1 } }, vars: { etcd_cluster: etcd } }
# minio cluster, optional backup repo for pgbackrest
#minio: { hosts: { 10.10.10.10: { minio_seq: 1 } }, vars: { minio_cluster: minio } }
# postgres cluster 'pg-meta' with single primary instance
pg-meta:
hosts: { 10.10.10.10: { pg_seq: 1, pg_role: primary } }
vars:
pg_cluster: pg-meta
pg_databases: [ { name: meta ,baseline: cmdb.sql ,comment: crystaldb meta database ,schemas: [ crystaldb ] ,extensions: [ { name: postgis, schema: public }, { name: timescaledb } ] } ]
pg_users:
- { name: dbuser_meta ,password: DBUser.Meta ,pgbouncer: true ,roles: [ dbrole_admin ] ,comment: crystaldb admin user }
- { name: dbuser_view ,password: DBUser.Viewer ,pgbouncer: true ,roles: [ dbrole_readonly ] ,comment: read-only viewer for meta database }
node_crontab: [ '00 01 * * * postgres /opt/CrystalDB/bin/pg-backup full' ] # make a full backup every 1am
vars: # global parameters
version: v2.2.0 # crystaldb version string
admin_ip: 10.10.10.10 # admin node ip address
region: default # upstream mirror region: default,china,europe
csi_portal: # domain names and upstream servers
home : { domain: h.crystaldb }
grafana : { domain: g.crystaldb ,endpoint: "${admin_ip}:3000" , websocket: true }
prometheus : { domain: p.crystaldb ,endpoint: "${admin_ip}:9090" }
alertmanager : { domain: a.crystaldb ,endpoint: "${admin_ip}:9093" }
blackbox : { endpoint: "${admin_ip}:9115" }
loki : { endpoint: "${admin_ip}:3100" }
#minio : { domain: sss.crystaldb ,endpoint: "${admin_ip}:9001" ,scheme: https ,websocket: true }
# if you want to use minio as backup repo instead of local fs, uncomment minio related lines
# don't forget to configure pgbackrest_repo and change credentials there!
#pgbackrest_method: minio
# if disabled, original /etc/yum.repos.d will be kept
repo_remove: false # remove existing repo on admin node during repo bootstrap
node_repo_remove: false # remove existing node repo for node managed by crystaldb
# WARNING: CHANGE THESE PASSWORDS
#grafana_admin_username: admin
grafana_admin_password: crystaldb
#pg_admin_username: dbuser_dba
pg_admin_password: DBUser.DBA
#pg_monitor_username: dbuser_monitor
pg_monitor_password: DBUser.Monitor
#pg_replication_username: replicator
pg_replication_password: DBUser.Replicator
#patroni_username: postgres
patroni_password: Patroni.API
#haproxy_admin_username: admin
haproxy_admin_password: crystaldb
# this config template assume you are using pre-packed offline packages
# If you wish to download upstream yum packages directly from internet,
# consider using ad hoc `el7.yml`, `el8.yml`, `el9.yml` config instead.
...
"""
with open("auto.yml", "w") as file:
file.write(auto)
with open("auto.yml", "w") as file:
inv = yaml.safe_load(auto)
invc = "pg-meta"
ip = "2.326.2.2"
hosts_ls = inv["all"]["children"][c]["hosts"]
if ip in hosts_ls:
print("IP already in this cluster")
max_seq = max([hosts_ls[i]["pg_seq"] for i in inv["all"]["children"][c]["hosts"]]) + 1
new_values = {'pg_seq': max_seq,
'pg_role': 'replica'}
inv["all"]["children"][c]["hosts"][ip] = new_values
inv["all"]["children"][c]["vars"]["pg_cluster"] = c
inv["all"]["children"][c]["hosts"]etcdc = "etcd"
ip = "23.326.2.2"
pgcluster = "etcd"
hosts_ls = inv["all"]["children"][etcdc]["hosts"]
if ip in hosts_ls:
print("IP already in this cluster")
max_seq = max([hosts_ls[i]["etcd_seq"] for i in inv["all"]["children"][etcdc]["hosts"]]) + 1
new_values = {'etcd_seq': max_seq}
inv["all"]["children"][etcdc]["hosts"][ip] = new_values
inv["all"]["children"][etcdc]["vars"]["etcd_cluster"] = etcdc
inv["all"]["children"][etcdc]def list_keys(data, parent_key='', separator='.'):
keys = []
if isinstance(data, dict):
for k, v in data.items():
new_key = f"{parent_key}{separator}{k}" if parent_key else k
keys.append(new_key)
keys += list_keys(v, new_key, separator=separator)
elif isinstance(data, list):
for i, v in enumerate(data):
new_key = f"{parent_key}[{i}]"
keys.append(new_key)
keys += list_keys(v, new_key, separator=separator)
return keys
# JSON example
import json
json_data = json.loads('{"a": {"b": 1, "c": 2}, "d": [3, 4]}')
print(list_keys(json_data))
# YAML example
import yaml
yaml_data = yaml.safe_load('a:\n b: 1\n c: 2\nd:\n - 3\n - 4')
print(list_keys(yaml_data)datawith open("crystaldb.yml", "r") as file:
data = yaml.safe_load(file)
cluster="pg-meta"
data["all"]["children"]list_keys(data)user_input = "pg-lucas"
user_new_ip = "1.1.1.1"
clusters = data["all"]["children"]
host_groups = list(clusters.keys())
# print(host_groups)
user_input in host_groups
clusters[user_input]
host_dict = clusters[user_input]["hosts"]
if not host_dict.get(user_new_ip):
for k, v in host_dict.items():
print(k+":", v)
print(list(v.keys()) if type(v) == dict else "")
host_dict[user_new_ip] = host_dict[k]
for i in host_dict[user_new_ip]:
if type(host_dict[user_new_ip][i]) == int:
host_dict[user_new_ip][i] = host_dict[user_new_ip][i] + 1
print(host_dict[user_new_ip])
elif host_dict[user_new_ip][i] == "primary":
host_dict[user_new_ip][i] = "replica"
print(host_dict)
[k for k,v in data["all"].items()]def list_keys(data, parent_key='', separator='.'):
keys = []
if isinstance(data, dict):
for k, v in data.items():
new_key = f"{parent_key}{separator}{k}" if parent_key else k
keys.append(new_key)
keys += list_keys(v, new_key, separator=separator)
elif isinstance(data, list):
for i, v in enumerate(data):
new_key = f"{parent_key}[{i}]"
keys.append(new_key)
keys += list_keys(v, new_key, separator=separator)
return keys
# JSON example
import json
json_data = json.loads('{"a": {"b": 1, "c": 2}, "d": [3, 4]}')
print(list_keys(json_data))
# YAML example
import yaml
yaml_data = yaml.safe_load('a:\n b: 1\n c: 2\nd:\n - 3\n - 4')
print(list_keys(yaml_data))