Files
2019-11-04 14:38:31 +00:00

768 lines
27 KiB
Python

import requests
import urllib3
import json
from typing import Union
import copy
from datetime import datetime
from dateutil.relativedelta import relativedelta
import os
import math
import time
from collections import OrderedDict
import re
from deprecated import deprecated
# warnings.filterwarnings('ignore')
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
class vropsAPI(object):
"""Module for the vropsAPI for Capacity Management:
Author: Daniel Tomlinson
Team: Capacity & Monitoring
Date: October 2019
VROPS documentation: https://sc1prapvro01/suite-api/docs/rest/
index.html
StatKeys for VMS:
https://docs.vmware.com/en/vRealize-Operations-Manager/6.7/com.vmware.vcom.metrics.doc/GUID-1322F5A4-DA1D-481F-BBEA-99B228E96AF2.html
Usage
-----
Authenticate a session:
vrops = vropsAPI.authenticate(
'https://sc1prapvro01/',
'username',
'authSource',
'password',
verify=False,
)
Get all clusters:
Query VROPS for available clusters:
vrops.getClusters()
vrops.getClusterIdentifiers()
Get dict of all cluster IDs and cluster names:
allClustersDict = vrops.allClusters
Get list of all cluster names:
allClustersList = vrops.getList(vrops.allClusters)
Get all hosts:
Query VROPS for available hosts:
From a single cluster:
vrops.getHostsFromCluster(cluster='SC1PRCONTXWHCUXCCL01')
vrops.getHostIdentifiers()
From a list of clusters:
<Not implemented>
Get dict of all host IDs and host names:
allHostsDisct =vrops.allHosts
Get list of host names:
allHostsList = vrops.getList(vrops.allHosts)
Get all VMs:
Query VROPS for available VMs:
For a single host:
vrops.getVMSFromHost('sc1hsesx148.prod.williamhill.plc')
vrops.getVMSIdentifiers()
For a list of hosts:
vrops.getVMSFromHost(allHostsList)
vrops.getVMSIdentifiers()
Get dict of all VM IDs and VM names:
allVMSDict = vrops.allVMS
Get list of all VMs:
allVMSList = vrops.getList(vrops.allVMS)
Get epoch time relative to another time:
Similar to Splunks relative_time command:
1. Can go back N hours/minutes etc.
2. Can set the hour/minute etc. to a specified value (snapping)
vrops.epochRelativeTime(epochTime, **kwargs)
**kwargs:
epochTime: int - start time
year: int = datetime.now().year # set year
month: int = datetime.now().month # set month
day: int = datetime.now().day # set day
hour: int = datetime.now().hour # set hour
minute: int = datetime.now().minute # set minute
second: int = datetime.now().second # set second
years: int = 0 # go back/forward N years
months: int = 0 # go back/forward N months
days: int = 0 # go back/forward N days
hours: int = 0 # go back/forward N hours
minutes: int = 0 # go back/forward N minutes
seconds: int = 0 # go back/forward N seconds
Usage:
Get epoch 5 minutes ago:
vrops.epochRelativeTime(vrops.epochNow, minutes=-5)
Get epoch at start of current hour:
vrops.epochRelativeTime(
vrops.epochNow,
hour=0,
minute=0,
second=0,
)
Get epoch 1 week ago at start of day:
vrops.epochRelativeTime(
vrops.epochNow,
days=-7
hour=0,
minute=0,
second=0,
)
Get stats from VMs:
Pull back results:
Last 30 minutes, 5 minute intervals, average
for CPU average and ready %:
vrops.getStatsFromVMS(
begin=vrops.epochRelativeTime(vrops.epochNow, minutes=-30),
end=vrops.epochNow,
intervalType='MINUTES',
intervalQuantifier='5',
rollUpType='AVG',
resourceId=list(vrops.allVMS.values()),
statKey=['cpu|usage_average', 'cpu|readyPct'],
)
Attributes:
intervalType: <ns3:intervalType> (see
https://sc1prapvro01/suite-api/docs/rest/
models.html#repr-1190589417)
intervalQuantifier: int
rollUpType: <ns3:rollUpType> (see:
https://sc1prapvro01/suite-api/
docs/rest/models.html#repr-1735704374)
resourceId: string or list of vrops resourceIds (not names)
statKey: vrops api metrics (see https://docs.vmware.com/en/
vRealize-Operations-Manager/6.7/com.vmware.vcom.metrics.doc/
GUID-1322F5A4-DA1D-481F-BBEA-99B228E96AF2.html)
Print results:
for i in range(0, vrops.totalVMS):
print(vrops.vmsResources['values'][i])
Save to disk as json:
vrops.saveToDisk(vrops.vmsResources)
Attributes
----------
vropsURL: str
URL of the VROPS instance
"https://sc1prapvro01/"
"""
defaultHeaders = {
'Accept': 'application/json',
'Content-Type': 'application/json',
'Cache-Control': 'no-cache',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive',
'cache-control': 'no-cache',
}
def __init__(self, vropsURL: str, authToken: str, verify: bool = True):
super(vropsAPI, self).__init__()
self.vropsURL = vropsURL
self.authToken = authToken
self.verify = verify
self.headers = vropsAPI.defaultHeaders
self.headers['Authorization'] = f'vRealizeOpsToken {self.authToken}'
@classmethod
def authenticate(
cls,
vropsURL: str,
username: str,
authSource: str,
password: str,
verify: bool = True,
):
vropsURLauth = vropsAPI.getVropsURL(vropsURL, 'authenticationURL')
payload = {}
for key, value in zip(
['username', 'authSource', 'password'],
[username, authSource, password],
):
payload[key] = value
authToken = vropsAPI.getAuthenticationToken(
vropsURLauth, payload, verify
)
return cls(vropsURL, authToken, verify)
@staticmethod
def getVropsURL(vropsURL: str, endpointKey: str) -> str:
endpoints = {
'authenticationURL': 'suite-api/api/auth/token/acquire',
'resourcesURL': 'suite-api/api/resources',
'statsURL': 'suite-api/api/resources/stats/query',
}
if endpoints[endpointKey] not in vropsURL:
if vropsURL[-1] != '/':
vropsURL = vropsURL + '/'
vropsURL = vropsURL + endpoints[endpointKey]
else:
vropsURL = vropsURL + endpoints[endpointKey]
return vropsURL
@staticmethod
def pythonToJSON(pythonObject: any, indent=4) -> str:
return json.dumps(pythonObject, indent=indent)
@staticmethod
def jsonToPython(jsonObject: str) -> any:
return json.loads(jsonObject)
@staticmethod
def getAuthenticationToken(
vropsURL: str, payload: dict, verify=True
) -> str:
payload = vropsAPI.pythonToJSON(payload)
vropsURL = vropsAPI.getVropsURL(vropsURL, 'authenticationURL')
response = requests.request(
'POST',
vropsURL,
data=payload,
headers=vropsAPI.defaultHeaders,
verify=verify,
)
print(response)
authToken = vropsAPI.jsonToPython(response.text)['token']
return authToken
@staticmethod
def getIdentifiers(
identifierDict: dict,
vropsJSON: dict,
length: int,
resourceKindKey: str,
) -> dict:
for i in range(0, length):
if (
vropsJSON['resourceList'][i]['resourceKey']['resourceKindKey']
== resourceKindKey
):
identifierDict[
vropsJSON['resourceList'][i]['resourceKey']['name']
] = vropsJSON['resourceList'][i]['identifier']
else:
pass
return identifierDict
@staticmethod
def getKeysList(pythonDict: dict) -> list:
pythonList = []
for i in pythonDict.keys():
pythonList.append(i)
return pythonList
@staticmethod
def getValuesList(pythonDict: dict) -> list:
pythonList = []
for i in pythonDict.values():
pythonList.append(i)
return pythonList
@staticmethod
def epochRelativeTime(
epochTime: int,
year: int = datetime.now().year,
month: int = datetime.now().month,
day: int = datetime.now().day,
hour: int = datetime.now().hour,
minute: int = datetime.now().minute,
second: int = datetime.now().second,
years: int = 0,
months: int = 0,
days: int = 0,
hours: int = 0,
minutes: int = 0,
seconds: int = 0,
) -> int:
delta = relativedelta(
year=year,
month=month,
day=day,
hour=hour,
minute=minute,
second=second,
years=years,
months=months,
days=days,
hours=hours,
minutes=minutes,
seconds=seconds,
)
if len(str(epochTime)) >= 12:
epochTime /= 1000
relativeTime = datetime.fromtimestamp(epochTime) + delta
relativeTime = math.ceil(relativeTime.timestamp() * 1000)
return relativeTime
def getClusters(self):
queryString = {'resourceKind': 'ClusterComputeResource'}
vropsURL = vropsAPI.getVropsURL(self.vropsURL, 'resourcesURL')
response = requests.request(
"GET",
vropsURL,
headers=self.headers,
params=queryString,
verify=self.verify,
)
print(response)
self._clusterResources = vropsAPI.jsonToPython(response.text)
return self
def getClusterIdentifiers(self):
self.totalClusters = len(self._clusterResources['resourceList'])
self.allClusters = {}
self.getIdentifiers(
self.allClusters,
self._clusterResources,
self.totalClusters,
'ClusterComputeResource',
)
return self
def getHostsFromCluster(self, cluster='SC1PRCONTXWHCUXCCL01'):
self.cluster = cluster
url = (
f'{self.vropsURL}/suite-api/api/resources/'
f'{self.allClusters[cluster]}/relationships'
)
self.chosenCluster = cluster
response = requests.request(
"GET", url, headers=self.headers, verify=self.verify
)
print(response)
self._hostResources = vropsAPI.jsonToPython(response.text)
return self
def getHostIdentifiers(self):
self.totalHosts = len(self._hostResources['resourceList'])
self.allHosts = {}
self.getIdentifiers(
self.allHosts, self._hostResources, self.totalHosts, 'HostSystem'
)
return self
def getVMSFromHost(self, host: Union[str, list]):
if isinstance(host, list):
self.vmType = 'list'
self._vmsResourcesRelationships = []
self.urlList = []
response = []
for item in host:
self.urlList.append(
(
f'{self.vropsURL}suite-api/api/resources/'
f'{self.allHosts[item]}/relationships'
)
)
for urlItem in self.urlList:
response.append(
requests.request(
'GET',
urlItem,
headers=self.headers,
verify=self.verify,
)
)
print(response)
for i in range(0, len(response)):
self._vmsResourcesRelationships.append(
vropsAPI.jsonToPython(response[i].text)
)
if isinstance(host, str):
hostToList = []
hostToList.append(host)
print(hostToList)
return self.getVMSFromHost(host=hostToList)
return self
def _getHostInformation(self, i: int, j: int):
pattern = r'(?:.*resources\/)([^\/]+)'
vm = self._vmsResourcesRelationships[i]['resourceList'][j][
'resourceKey'
]['name']
host = self.urlList[i]
match = re.findall(pattern, host)
for key, value in self.allHosts.items():
if match[0] == value:
self.VMSHostsNames[vm] = key
return self
def getVMSIdentifiers(self):
self.VMSHostsNames = {}
self.allVMS = OrderedDict()
if self.vmType == 'list':
self.countVMS = []
self.countVMSFiltered = []
for i in range(0, len(self._vmsResourcesRelationships)):
counter = 0
for j in range(
0, len(self._vmsResourcesRelationships[i]['resourceList'])
):
if (
self._vmsResourcesRelationships[i]['resourceList'][j][
'resourceKey'
]['resourceKindKey']
) == 'VirtualMachine':
counter += 1
self._getHostInformation(i, j)
self.countVMS.append(
len(self._vmsResourcesRelationships[i]['resourceList'])
)
self.countVMSFiltered.append(counter)
for i in range(0, len(self._vmsResourcesRelationships)):
self.getIdentifiers(
self.allVMS,
self._vmsResourcesRelationships[i],
self.countVMS[i],
'VirtualMachine',
)
if self.vmType == 'string':
counter = 0
self.countVMS = len(
self._vmsResourcesRelationships['resourceList']
)
for j in range(0, self.countVMS):
if (
self._vmsResourcesRelationships['resourceList'][j][
'resourceKey'
]['resourceKindKey']
) == 'VirtualMachine':
counter += 1
self.countVMSFiltered = counter
self.getIdentifiers(
self.allVMS,
self._vmsResourcesRelationships,
self.countVMS,
'VirtualMachine',
)
return self
def getStats(
self,
begin: int,
end: int,
intervalType: str,
intervalQuantifier: str,
rollUpType: str,
resourceId: list,
statKey: Union[str, list],
vropsType: str,
*args,
**kwargs,
):
self.vropsType = vropsType
argList = copy.deepcopy(locals())
for i in ('self', 'vropsType'):
del argList[i]
vropsURL = self.getVropsURL(self.vropsURL, 'statsURL')
payload = self.pythonToJSON(argList, indent=0)
response = requests.request(
'POST',
vropsURL,
headers=self.headers,
data=payload,
verify=self.verify,
)
self._vmsResources = OrderedDict(self.jsonToPython(response.text))
if self.vropsType == 'virtualmachine':
for key, value in self.allVMS.items():
for i in range(0, len(self._vmsResources['values'])):
if self._vmsResources['values'][i]['resourceId'] == value:
self._vmsResources['values'][i] = OrderedDict(
self._vmsResources['values'][i]
)
self._vmsResources['values'][i]['name'] = key
self._vmsResources['values'][i][
'host'
] = self.VMSHostsNames[key]
self._vmsResources['values'][i][
'cluster'
] = self.chosenCluster
for item in ['cluster', 'host', 'name']:
self._vmsResources['values'][i].move_to_end(
item, last=False
)
if self.vropsType == 'host':
for key, value in self.allHosts.items():
for i in range(0, len(self._vmsResources['values'])):
if self._vmsResources['values'][i]['resourceId'] == value:
self._vmsResources['values'][i] = OrderedDict(
self._vmsResources['values'][i]
)
self._vmsResources['values'][i]['name'] = key
self._vmsResources['values'][i][
'cluster'
] = self.chosenCluster
for item in ['cluster', 'name']:
self._vmsResources['values'][i].move_to_end(
item, last=False
)
@deprecated(
version='2.1',
reason='this method is deprecated, use getStats() instead with the'
' same functionality',
)
def getStatsFromVMS(
self,
begin: int,
end: int,
intervalType: str,
intervalQuantifier: str,
rollUpType: str,
resourceId: list,
statKey: Union[str, list],
vropsType: str,
):
argList = copy.deepcopy(locals())
del argList['self']
self.getStats(**argList)
return self
@staticmethod
def saveToDisk(
pythonObject: any,
path: str = os.getcwd(),
filePrefix: str = '',
type: str = 'json',
indent: int = 4,
breakLine: bool = False,
) -> None:
timeNow = datetime.now().strftime('%d-%m-%Y_%H-%M-%S')
fileName = f'{path}/{filePrefix}-{timeNow}.json'
if breakLine:
if not isinstance(pythonObject, list):
raise TypeError(
'You must pass a list when using' ' breakLine=True'
)
else:
with open(fileName, 'a+') as outputFile:
for item in pythonObject:
try:
outputFile.write(
json.dump(
json.loads(item), outputFile, indent=indent
)
)
except TypeError:
pass
outputFile.write('\n')
else:
with open(fileName, 'w+') as outputFile:
json.dump(pythonObject, outputFile, indent=indent)
def exportVMData(self):
self.export = []
loopLength = len(self._vmsResources['values'])
for i in range(0, loopLength):
statKeyLength = len(
self._vmsResources['values'][i]['stat-list']['stat']
)
timeLength = len(
self._vmsResources['values'][i]['stat-list']['stat'][0][
'timestamps'
]
)
for k in range(0, statKeyLength):
for j in range(0, timeLength):
if self.vropsType == 'virtualmachine':
self.export.append(
json.dumps(
{
'type': self.vropsType,
'name': self._vmsResources['values'][i][
'name'
],
'host': self._vmsResources['values'][i][
'host'
],
'cluster': self.chosenCluster,
'timestamp': self._vmsResources['values'][
i
]['stat-list']['stat'][0]['timestamps'][j],
'value': self._vmsResources['values'][i][
'stat-list'
]['stat'][k]['data'][j],
'statKey': self._vmsResources['values'][i][
'stat-list'
]['stat'][k]['statKey']['key'],
'rollUpType': self._vmsResources['values'][
i
]['stat-list']['stat'][k]['rollUpType'],
'intervalQuantifier': self._vmsResources[
'values'
][i]['stat-list']['stat'][k][
'intervalUnit'
][
'quantifier'
],
'intervalType': self._vmsResources[
'values'
][i]['stat-list']['stat'][0][
'intervalUnit'
][
'intervalType'
],
}
)
)
elif self.vropsType == 'host':
self.export.append(
json.dumps(
{
'type': self.vropsType,
'name': self._vmsResources['values'][i][
'name'
],
'cluster': self.chosenCluster,
'timestamp': self._vmsResources['values'][
i
]['stat-list']['stat'][0]['timestamps'][j],
'value': self._vmsResources['values'][i][
'stat-list'
]['stat'][k]['data'][j],
'statKey': self._vmsResources['values'][i][
'stat-list'
]['stat'][k]['statKey']['key'],
'rollUpType': self._vmsResources['values'][
i
]['stat-list']['stat'][k]['rollUpType'],
'intervalQuantifier': self._vmsResources[
'values'
][i]['stat-list']['stat'][k][
'intervalUnit'
][
'quantifier'
],
'intervalType': self._vmsResources[
'values'
][i]['stat-list']['stat'][0][
'intervalUnit'
][
'intervalType'
],
}
)
)
return self
@property
def totalVMS(self):
if isinstance(self.countVMSFiltered, list):
self.__totalVMS = sum(self.countVMSFiltered)
elif isinstance(self.countVMSFiltered, int):
self.__totalVMS = self.countVMSFiltered
return self.__totalVMS
@totalVMS.setter
def totalVMS(self, totalVMS):
self.__totalVMS = totalVMS
return self.__totalVMS
# @property
# def totalHosts(self):
# if isinstance(self.countVMSFiltered, list):
# self.__totalVMS = sum(self.countVMSFiltered)
# elif isinstance(self.countVMSFiltered, int):
# self.__totalVMS = self.countVMSFiltered
# return self.__totalVMS
# @totalHosts.setter
# def totalHosts(self, totalHosts):
# self.__totalHosts = totalHosts
# return self.__totalHosts
@property
def epochNow(self):
self.__epochNow = math.ceil(time.time() * 1000)
return self.__epochNow
@property
def epochToday(self):
now = datetime.now()
self.__epochtoday = now + relativedelta(hour=0, minute=0, second=0)
self.__epochToday = math.ceil(time.time() * 1000)
return self.__epochToday
@property
def allVMS(self):
return self.__allVMS
@allVMS.setter
def allVMS(self, allVMS):
if not isinstance(allVMS, dict):
raise TypeError(
'You must pass a dictionary with a key of the name'
f' and a value of the VROPS ID, not {type(allVMS)}.'
)
else:
pass
self.__allVMS = allVMS
print('Successfully imported the dictionary.')
return self.__allVMS
@property
def VMSHostsNames(self):
return self.__VMSHostsNames
@VMSHostsNames.setter
def VMSHostsNames(self, VMSHostsNames):
if not isinstance(VMSHostsNames, dict):
raise TypeError(
'You must pass a dictionary with a key of the name'
f' and a value of the VROPS ID, not {type(VMSHostsNames)}.'
)
else:
pass
self.__VMSHostsNames = VMSHostsNames
print('Successfully imported the dictionary.')
return self.__VMSHostsNames
@property
def chosenCluster(self):
return self.__chosenCluster
@chosenCluster.setter
def chosenCluster(self, chosenCluster):
if not isinstance(chosenCluster, str):
raise TypeError(
'You must pass a dictionary with a key of the name'
f' and a value of the VROPS ID, not {type(chosenCluster)}.'
)
else:
pass
self.__chosenCluster = chosenCluster
return self.__chosenCluster