工作使用場景

工作中需要透過CPU、記憶體生成資源利用率報表，可以透過Prometheus的API寫一個Python指令碼。

可透過API獲取資料，然後再進行資料排序、過濾、運算、聚合，最後寫入Mysql資料庫。

CPU峰值計算

取最近一週CPU數值，再排序取最高的值。

def get_cpu_average(self): """ CPU忙時平均值：取最近一週CPU資料，每一天的TOP20除以20得到忙時平均值；再將一週得到的忙時平均值相加，再除以7，得到時間範圍內一週的忙時平均值。 :return: """ cpu_average = {} for t in range(len(self.time_list)): if t + 1 < len(self.time_list): start_time = self.time_list[t] end_time = self. time_list[t + 1] # print(start_time, end_time) # 拼接URL pre_url = server_ip + '/api/v1/query_range?query=' expr = '100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) ' \ '&start=%s&end=%s&step=300' % (start_time, end_time - 1) url = pre_url + expr # print(url) # 請求介面資料 data = json.loads(requests.post(url=url, headers=self.headers).content.decode('utf8', 'ignore')) for da in data.get('data').get('result'): # 迴圈拿到result資料 values = da.get('values') cpu_load = [float(v[1]) for v in values] # 迴圈拿到values裡面的所有值 ip = da.get('metric').get('instance') # 拿到instance裡面的ip ip = ip[:ip.index(':')] if ':' in ip else ip # 去除個別後面帶的埠號 # avg_cup_load = sum(sorted(cpu_load, reverse=True)[:20]) / 20 # 取top20% 再除以20%，得出top20%的平均值 # avg_cup_load = round(sum(sorted(cpu_load, reverse=True)[:round(len(cpu_load) * 0.2)]) / round(len(cpu_load) * 0.2), 2) # 倒序後取前面20%除以個數，得到前20%的平均值 avg_cup_load = sum(sorted(cpu_load, reverse=True)[:round(len(cpu_load) * 0.2)]) / round(len(cpu_load) * 0.2) # print(avg_cup_load) # 將計算後的資料以ip為key寫入字典 if cpu_average.get(ip): cpu_average[ip].append(avg_cup_load) else: cpu_average[ip] = [avg_cup_load] # 每日top20的平均值累加，共7天的再除以7 for k, v in cpu_average.items(): # cpu_average[k] = round(sum(v) / 7, 2) cpu_average[k] = sum(v) # print(cpu_average) return cpu_average cpu_average[ip] = [avg_cup_load] # 每日top20的平均值累加，共7天的再除以7 for k, v in cpu_average.items(): # cpu_average[k] = round(sum(v) / 7, 2) cpu_average[k] = sum(v) # print(cpu_average) return cpu_average cpu_average[ip] = [avg_cup_load] # 每日top20的平均值累加，共7天的再除以7 for k, v in cpu_average.items(): # cpu_average[k] = round(sum(v) / 7, 2) cpu_average[k] = sum(v) # print(cpu_average) return cpu_average

記憶體峰值計算

取7天記憶體數值，排序後取最高峰值

def get_mem_average(self): """ 記憶體忙時平均值：先取出7天的日期，根據多條連結迴圈取出每天資料，排序value取top20除以20，最終7天資料再除以7 :return: """ avg_mem_util = {} for t in range(len(self.time_list)): if t + 1 < len(self.time_list): start_time = self.time_list[t] end_time = self.time_list[t + 1] # 根據多條連結迴圈取出每天資料 pre_url = self.server_ip + '/api/v1/query_range?query=' # expr = '(node_memory_MemTotal_bytes - (node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes )) / node_memory_MemTotal_bytes * 100&start=%s&end=%s&step=300' % (start_time, end_time) expr_MenTotal = 'node_memory_MemTotal_bytes&start=%s&end=%s&step=600' % (start_time, end_time - 1) expr_MemFree = 'node_memory_MemFree_bytes&start=%s&end=%s&step=600' % (start_time, end_time - 1) expr_Buffers = 'node_memory_Buffers_bytes&start=%s&end=%s&step=600' % (start_time, end_time - 1) expr_Cached = 'node_memory_Cached_bytes&start=%s&end=%s&step=600' % (start_time, end_time - 1) result = {} # 迴圈取出四個欄位 for ur in expr_MenTotal, expr_MemFree, expr_Buffers, expr_Cached: url = pre_url + ur data = json.loads(requests.post(url=url, headers=self.headers).content.decode('utf8', 'ignore')) ip_dict = {} # 迴圈單個欄位所有值 for da in data.get('data').get('result'): ip = da.get('metric').get('instance') ip = ip[:ip.index(':')] if ':' in ip else ip if ip_dict.get(ip): # print("重複ip：%s" % (ip)) continue values = da.get('values') # 將列表裡的值轉為字典方便計算 values_dict = {} for v in values: values_dict[str(v[0])] = v[1] # 標記ip存在 ip_dict[ip] = True # 建立列表追加字典 if result.get(ip): result[ip].append(values_dict) else: result[ip] = [values_dict] # print(result) for ip, values in result.items(): values_list = [] for k, v in values[0].items(): try: values_MenTotal = float(v) values_MemFree = float(values[1].get(k, 0)) if values[1] else 0 values_Buffers = float(values[2].get(k, 0)) if values[2] else 0 values_Cached = float(values[3].get(k, 0)) if values[3] else 0 if values_MemFree == 0.0 or values_Buffers == 0.0 or values_Cached == 0.0: continue value_calc = (values_MenTotal - (values_MemFree + values_Buffers + values_Cached)) / values_MenTotal * 100 if value_calc != float(0): values_list.append(value_calc) except Exception as e: print(values[0]) # logging.exception(e) continue # 排序value取top20除以20 # avg_mem = round(sum(sorted(values_list, reverse=True)[:round(len(values_list) * 0.2)]) / round(len(values_list) * 0.2), 2) try: avg_mem = sum(sorted(values_list, reverse=True)[:round(len(values_list) * 0.2)]) / round(len(values_list) * 0.2) except Exception as e: avg_mem = 0 logging.exception(e) if avg_mem_util.get(ip): avg_mem_util[ip].append(avg_mem) else: avg_mem_util[ip] = [avg_mem] # 最終7天資料再除以7 for k, v in avg_mem_util.items(): # avg_mem_util[k] = round(sum(v) / 7, 2) avg_mem_util[k] = sum(v) return avg_mem_util

匯出excel

將採集到的資料匯出excel

<pre class="hljs python" "="" style="overflow: auto; font-family: Menlo, Monaco, Consolas, "Courier New", monospace; font-size: 13px; padding: 0.5em; margin-top: 0px; margin-bottom: 10px; line-height: 1.42857; color: rgb(68, 68, 68); word-break: break-all; overflow-wrap: normal; background: rgb(240, 240, 240); border: 1px solid rgb(204, 204, 204); border-radius: 4px; max-width: ;">def export_excel(self, export): """ 將採集到的資料匯出excel :param export: 資料集合 :return: """ try: # 將字典列表轉換為DataFrame pf = pd.DataFrame(list(export)) # 指定欄位順序 order = ['ip', 'cpu_peak', 'cpu_average', 'mem_peak', 'mem_average', 'collector'] pf = pf[order] # 將列名替換為中文 columns_map = { 'ip': 'ip', 'cpu_peak': 'CPU峰值利用率', 'cpu_average': 'CPU忙時平均峰值利用率', 'mem_peak': '記憶體峰值利用率', 'mem_average': '記憶體忙時平均峰值利用率', 'collector': '來源地址' } pf.rename(columns=columns_map, inplace=True) # 指定生成的Excel表格名稱 writer_name = self.Host + '.xlsx' writer_name.replace(':18600', '') # print(writer_name) file_path = pd.ExcelWriter(writer_name.replace(':18600', '')) # 替換空單元格 pf.fillna(' ', inplace=True) # 輸出 pf.to_excel(file_path, encoding='utf-8', index=False) # 儲存表格 file_path.save() except Exception as e: print(e) logging.exception(e)

Meta佛薩奇矩陣模式/MetaForce/系統開發/原始碼搭建方案/魔豹聯盟開發技術講解

相關文章