VMware vSphere Monitoring

/*******************************************************************************
 * © 2007-2024 - LogicMonitor, Inc. All rights reserved.
 ******************************************************************************/

import com.santaba.agent.groovy.utils.GroovyScriptHelper
import com.logicmonitor.mod.Snippets
import com.santaba.agent.AgentVersion
import java.text.DecimalFormat

// To run in debug mode, set to true
Boolean debug = false
// To enable logging, set to true
Boolean log = false

// Set props object based on whether or not we are running inside a netscan or debug console
def props
try {
    hostProps.get("system.hostname")
    props = hostProps
    debug = true  // set debug to true so that we can ensure we do not print sensitive properties
}
catch (MissingPropertyException) {
    props = netscanProps
}

def host = props.get("vcenter.hostname", props.get("system.hostname"))
def user = props.get("vcenter.user") ?: props.get("vcsa.user") ?: props.get("esx.user")
def pass = props.get("vcenter.pass") ?: props.get("vcsa.pass") ?: props.get("esx.pass")
def addr = props.get("vcenter.url") ?: props.get("vcsa.url") ?: props.get("esx.url") ?: "https://${host}/sdk"
if (!host || !user || !pass || !addr) {
    throw new Exception("Required parameters are missing.")
}

def logCacheContext = "${host}::vmware-vsphere"
Boolean skipDeviceDedupe = props.get("skip.device.dedupe", "false").toBoolean()
String hostnameSource    = props.get("hostname.source", "")?.toLowerCase()?.trim()

Integer collectorVersion = AgentVersion.AGENT_VERSION.toInteger()
 
// Bail out early if we don't have the correct minimum collector version to ensure netscan runs properly
if (collectorVersion < 32400) {
    def formattedVer = new DecimalFormat("00.000").format(collectorVersion / 1000)
    throw new Exception("Upgrade collector running netscan to 32.400 or higher to run full featured enhanced netscan. Currently running version ${formattedVer}.")
}

props["proxy.enable"] = false

def modLoader = GroovyScriptHelper.getInstance(GroovySystem.version).getScript("Snippets", Snippets.getLoader()).withBinding(getBinding())
def emit        = modLoader.load("lm.emit", "1.1")
def lmDebugSnip = modLoader.load("lm.debug", "1")
def lmDebug     = lmDebugSnip.debugSnippetFactory(out, debug, log, logCacheContext)
def httpSnip    = modLoader.load("proto.http", "0")
def http        = httpSnip.httpSnippetFactory(props)
def cacheSnip   = modLoader.load("lm.cache", "0")
def cache       = cacheSnip.cacheSnippetFactory(lmDebug, logCacheContext)
def vSphereAPISnip = modLoader.load("vmware.vsphere.api")
def vSphereAPI = vSphereAPISnip.vSphereAutomationAPIFactory(props, lmDebug, cache, http)
def vSphereSDKSnip = modLoader.load("vmware.vsphere.sdk")
def vSphereSDK = vSphereSDKSnip.vSphereAutomationSDKFactory(props, lmDebug, cache)

// Only initialize lmApi snippet class if customer has not opted out
def lmApi
if (!skipDeviceDedupe) {
    def lmApiSnippet = modLoader.load("lm.api", "0")
    lmApi = lmApiSnippet.lmApiSnippetFactory(props, http, lmDebug)
}

def esxUser = props.get("esx.user", user)
def esxPass = props.get("esx.pass", pass)
def rootFolder = props.get("rootFolder") ?: "vCenterResources"
def includeESXiHosts = (props.get("discover.esxi") ?: true).toBoolean()
def includeVMs = (props.get("discover.vm") ?: true).toBoolean()
def includeHostsAndClustersView = (props.get("view.hostandcluster") ?: true).toBoolean()
def includeVMsAndTemplatesView = (props.get("view.vmsandtemplates") ?: true).toBoolean()
def includeStandaloneVM = (props.get("view.standaloneVm") ?: true).toBoolean()

def maxESXiLoginFailure = (props.get("esx.maxloginfailures", "10").trim()).toInteger()
int eTimeout = 3
int ttlESXiLoginFailure = 0

// Get information about devices that already exist in LM portal
List fields = ["name", "currentCollectorId", "displayName"]
Map args = ["size": 1000, "fields": fields.join(",")]
def lmDevices
// But first determine if the portal size is within a range that allows us to get all devices at once
def pathFlag, portalInfo, timeLimitSec, timeLimitMs
if (!skipDeviceDedupe) {
    portalInfo = lmApi.apiCallInfo("Devices", args)
    timeLimitSec = props.get("lmapi.timelimit.sec", "60").toInteger()
    timeLimitMs = (timeLimitSec) ? Math.min(Math.max(timeLimitSec, 30), 120) * 1000 : 60000 // Allow range 30-120 sec if configured; default to 60 sec

    if (portalInfo.timeEstimateMs > timeLimitMs) {
        lmDebug.LMDebugPrint("Estimate indicates LM API calls would take longer than time limit configured.  Proceeding with individual queries by display name for each device to add.")
        lmDebug.LMDebugPrint("\t${portalInfo}\n\tNOTE:  Time limit is set to ${timeLimitSec} seconds.  Adjust this limit by setting the property lmapi.timelimit.sec.  Max 120 seconds, min 30 seconds.")
        pathFlag = "ind"
    }
    else {
        lmDebug.LMDebugPrint("Response time indicates LM API calls will complete in a reasonable time range.  Proceeding to collect info on all devices to cross reference and prevent duplicate device creation.\n\t${portalInfo}")
        pathFlag = "all"
        lmDevices = lmApi.getPortalDevices(args)
    }
}

List<Map> resources = []

def now = new Date()
def dateFormat = "yyyy-MM-dd'T'HH:mm:ss.s z"
TimeZone tz = TimeZone.getDefault()
Map duplicateResources = [
    "date" : now.format(dateFormat, tz),
    "message" : "Duplicate device names and display names, keyed by display name that would be assigned by the netscan, found within LogicMonitor portal.  Refer to documentation for how to resolve name collisions using 'hostname.source' netscan property.",
    "total" : 0,
    "resources" : []
]

def ilpCredentials = [:]
def tags = [:]
// Determine the vSphere host that we are making API queries to
def productType = ["vpx": 0, "embeddedEsx": 1]
def productId = productType[vSphereSDK.si.serviceContent.about.getProperties().productLineId]
if (productId == 0) {  // vCenter
    ilpCredentials["vcenter.user"] = user
    ilpCredentials["vcenter.pass"] = pass
    ilpCredentials["vcenter.addr"] = addr
    tags = vSphereAPI.getTags()
    lmDebug.LMDebugPrint("INFO: vCenter tags found: ${tags}")
} else if (productId == 1) { // ESXi
    ilpCredentials["esx.user"] = esxUser ?: user
    ilpCredentials["esx.pass"] = esxPass ?: pass
    ilpCredentials["esx.addr"] = addr
}

def tagPropPrefix = "vmware.vcenter.tag"
Closure tagFormatter = { (it) ? it.collectEntries { cat, tag -> ["${tagPropPrefix}.${cat}", tag.join(",")] } : [:] }

def foundHost = false
def relationships = [:]
def vms = vSphereSDK.getMOs("VirtualMachine").each { updateEntityLineage(it, relationships) }
def hosts = vSphereSDK.getMOs("HostSystem").each { updateEntityLineage(it, relationships) }
vms.each { vm ->
    def resource = [:]
    def MOR = vm.MOR
    if (!MOR) {
        lmDebug.LMDebug("ERROR: MOR not found for ${vm}, skipping")
        return
    }
    if (vm.resourcePool == null) {
        lmDebug.LMDebug("ERROR: ${vm} is a template, skipping")
        return       
    }

    String displayName = vm.name

    def hostname = displayName
    def ip = displayName
    def vmMo = vSphereSDK.createMO(MOR)
    def guestIpAddress = vmMo?.guest?.ipAddress
    if (guestIpAddress) {
        ip = guestIpAddress
        hostname = ip
        def reverseDnsName = InetAddress.getByName(guestIpAddress)?.getHostName()
        if (reverseDnsName) {
            hostname = reverseDnsName
        }
    }

    // Check for existing device in LM portal with this displayName; set to false initially and update to true when dupe found
    def deviceMatch = false
    // If customer has opted out of device deduplication checks, we skip the lookups where we determine if a match exists and proceed as false
    if (!skipDeviceDedupe) {
        if (pathFlag == "ind") {
            deviceMatch = lmApi.findPortalDevice(displayName, args)
            if (!deviceMatch) deviceMatch = lmApi.findPortalDeviceByName(ip, args)
        }
        else if (pathFlag == "all") {
            deviceMatch = lmApi.checkExistingDevices(displayName, lmDevices)
            if (!deviceMatch) deviceMatch = lmApi.checkExistingDevicesByName(ip, lmDevices)
        }
    }
    if (deviceMatch) {
        // Log duplicates that would cause additional devices to be created; unless these entries are resolved, they will not be added to resources for netscan output
        def collisionInfo = [
            (displayName) : [
                "Netscan" : [
                    "hostname"    : ip,
                    "displayName" : displayName
                ],
                "LM" : [
                    "hostname"    : deviceMatch.name,
                    "collectorId" : deviceMatch.currentCollectorId,
                    "displayName" : deviceMatch.displayName
                ],
                "Resolved" : false
            ]
        ]

        // If user specified to use LM hostname on display name match, update hostname variable accordingly
        // and flag it as no longer a match since we have resolved the collision with user"s input
        if (hostnameSource == "lm" || hostnameSource == "logicmonitor") {
            ip = deviceMatch.name
            deviceMatch = false
            collisionInfo[displayName]["Resolved"] = true
        }
        // If user specified to use netscan data for hostname, update the display name to make it unique
        // and flag it as no longer a match since we have resolved the collision with user"s input
        else if (hostnameSource == "netscan") {
            // Update the resolved status before we change the displayName
            collisionInfo[displayName]["Resolved"] = true
            displayName = "${displayName} - ${ip}"
            deviceMatch = false
        }

        duplicateResources["resources"].add(collisionInfo)
    }

    foundHost = vSphereSDK.isVmTheHost(vmMo)
    if (!includeVMs && foundHost != vmMo) return

    def deviceGroups = []
    def compLine = getLineage("RESOURCEPOOL", MOR.val, relationships)
    def vmLine = getLineage("FOLDER", MOR.val, relationships)
    def compFolder = includeHostsAndClustersView ? folderFormatter(compLine, rootFolder, includeStandaloneVM) : null
    def vmFolder = includeVMsAndTemplatesView ? folderFormatter(vmLine, rootFolder, includeStandaloneVM) : null
    if (foundHost == vmMo) {
        // Rely on the vmFolder here. If not including standalone VMs, but the vCenter is a standalone VM, discover it anyway
        def baseFolder = (vmFolder ?: folderFormatter(vmLine, rootFolder, includeStandaloneVM))?.split("/")[0]
        deviceGroups << baseFolder
    }
    if (includeVMs) deviceGroups << compFolder << vmFolder
    deviceGroups = deviceGroups.minus(null)
    if (!deviceGroups) {
        lmDebug.LMDebug("DEBUG: VM ${vm.name} skipped due to no containing folder determined. This could be due to different filters being set.")
        return
    }

    if (foundHost == vmMo) { // This is the vCenter VM
        resource = [
            "hostname"   : hostname,
            "displayname": displayName,
            "hostProps"  : ["system.categories": "VMware_vCenter,VMwareVM",
                            "vcenter.vm.name"  : vm.name,
                            "vcenter.hostname" : host,
                            "esx.user"         : esxUser,
                            "esx.pass"         : esxPass],
            "groupName"  : deviceGroups << "VMware vCenters"
        ]
        resource.hostProps += tagFormatter(tags[vSphereSDK.rootFolder.MOR.value])
        resource.hostProps += ilpCredentials
    } else {
        resource = [
            "hostname"   : hostname,
            "displayname": displayName,
            "hostProps"  : ["system.categories": "VMwareVM",
                            "vcenter.vm.name"  : vm.name],
            "groupName"  : deviceGroups
        ]
    }

    // Power state for filtering
    def powerState = vm.runtime?.powerState?.value?.toString()
    if (powerState) resource.hostProps += ["netscan.powerstate": powerState]

    // If the netscan is running on a vCenter device
    if (productId == 0) {
        // Add vCenter tags
        resource.hostProps += tagFormatter(tags[MOR.value as String])

        // Add all parent entities as device properties
        def lineage = lineageParser(compLine + vmLine)
        lineage.each { k, v -> resource.hostProps?."vcenter.${k.toLowerCase()}" = v.join(",") }
    }

    resources.add(resource)
}

if (includeESXiHosts) {
    hosts.each { esxi ->
        def resource = [:]
        def MOR = esxi.MOR
        if (!MOR) {
            lmDebug.LMDebug("ERROR: MOR not found for ${esxi}, skipping")
            return
        }

        String displayName = esxi.name
        
        def hostname = displayName
        def ip = displayName
        def esxiMo = vSphereSDK.createMO(MOR)
        def hostIpAddress = esxiMo?.config?.vmotion?.ipConfig?.ipAddress
        if (hostIpAddress) {
            ip = hostIpAddress
            hostname = ip
            def reverseDnsName = InetAddress.getByName(hostIpAddress)?.getHostName()
            if (reverseDnsName) {
                hostname = reverseDnsName
            }
        }

        // Check for existing device in LM portal with this displayName; set to false initially and update to true when dupe found
        def deviceMatch = false
        // If customer has opted out of device deduplication checks, we skip the lookups where we determine if a match exists and proceed as false
        if (!skipDeviceDedupe) {
            if (pathFlag == "ind") {
                deviceMatch = lmApi.findPortalDevice(displayName, args)
                if (!deviceMatch) deviceMatch = lmApi.findPortalDeviceByName(ip, args)
            }
            else if (pathFlag == "all") {
                deviceMatch = lmApi.checkExistingDevices(displayName, lmDevices)
                if (!deviceMatch) deviceMatch = lmApi.checkExistingDevicesByName(ip, lmDevices)
            }
        }
        if (deviceMatch) {
            // Log duplicates that would cause additional devices to be created; unless these entries are resolved, they will not be added to resources for netscan output
            def collisionInfo = [
                (displayName) : [
                    "Netscan" : [
                        "hostname"    : ip,
                        "displayName" : displayName
                    ],
                    "LM" : [
                        "hostname"    : deviceMatch.name,
                        "collectorId" : deviceMatch.currentCollectorId,
                        "displayName" : deviceMatch.displayName
                    ],
                    "Resolved" : false
                ]
            ]

            // If user specified to use LM hostname on display name match, update hostname variable accordingly
            // and flag it as no longer a match since we have resolved the collision with user"s input
            if (hostnameSource == "lm" || hostnameSource == "logicmonitor") {
                ip = deviceMatch.name
                deviceMatch = false
                collisionInfo[displayName]["Resolved"] = true
            }
            // If user specified to use netscan data for hostname, update the display name to make it unique
            // and flag it as no longer a match since we have resolved the collision with user"s input
            else if (hostnameSource == "netscan") {
                // Update the resolved status before we change the displayName
                collisionInfo[displayName]["Resolved"] = true
                displayName = "${displayName} - ${ip}"
                deviceMatch = false
            }

            duplicateResources["resources"].add(collisionInfo)
        }

        def deviceGroups = []
        def compLine = getLineage("RESOURCEPOOL", MOR.val, relationships)
        def compFolder = folderFormatter(compLine, rootFolder, includeStandaloneVM)
        deviceGroups << "${compFolder}/ESXi hosts"
        deviceGroups = deviceGroups.minus(null)

        def vSphere = null
        def esxiAddr = "https://${esxi.name}/sdk"
        lmDebug.LMDebug("INFO: Attempting to login to the ESX host ${esxi} (${esxi.name}) with user ${esxUser}")
        try {
            if (ttlESXiLoginFailure < maxESXiLoginFailure) {
                vSphere = vSphereSDKSnip.vSphereAutomationSDKFactory(props, lmDebug, cache, hostname)
            } else {
                vSphere = null
            }
        } catch (Exception e) {
            lmDebug.LMDebug("\tDEBUG: Unable to login to ESXi host ${esxi} at ${esxiAddr}, skipping...")
            lmDebug.LMDebug("\t\tERROR: $e")
            ttlESXiLoginFailure++
        }
        if (!vSphere) {
            lmDebug.LMDebug("DEBUG: Unable to connect to ESXi host ${esxi.name} with the provided credentials")

            // We were unable to connect to the ESXi host directly, just add it as a standard device
            resource = [
                "hostname"   : hostname,
                "displayname": displayName,
                "hostProps"  : ["esx.netscan.status": "Netscan did not connect to ESX host."], // Do not set the system category "VMware_ESXi" since we can't actually use this as an ESXi device
                "groupName"  : deviceGroups
            ]
        } else {
            lmDebug.LMDebug("INFO: Successful connection to ESXi host ${esxi.name}")
            def esxMo = vSphereSDK.getMOs("HostSystem")[0]
            resource = [
                "hostname"   : hostname,
                "displayname": displayName,
                "hostProps"  : [
                                "esx.user"         : esxUser,
                                "esx.pass"         : esxPass,
                                "esx.addr"         : esxiAddr,
                                "system.categories": "VMware_ESXi"],
                "groupName"  : deviceGroups
            ]
        }

        resource.hostProps."vcenter.hostname" = host
        resource.hostProps += ilpCredentials

        // If the netscan is running on a vCenter device
        if (productId == 0) {
            // Add vCenter tags
            resource.hostProps += tagFormatter(tags[MOR?.value as String])

            // Add all parent entities as device properties
            def lineage = lineageParser(compLine)
            lineage.each { k, v -> resource.hostProps?."vcenter.${k.toLowerCase()}" = v.join(",") }
        }

        resources.add(resource)
    }
}

lmDebug.LMDebugPrint("Duplicate Resources:")
duplicateResources.resources.each {
    lmDebug.LMDebugPrint("\t${it}")
}

emit.resource(resources, debug)

return 0

def folderFormatter(lineage, rootFolder = "", includeStandaloneVM) {
    if (lineage.size() > 1) {
        def folder = lineage.collect { Map folderMap ->
            def type = folderMap.type.toUpperCase()
            if (type == "FOLDER") {
                if (folderMap.parent[0] == null) { // This is the parent folder
                    return rootFolder ?: "VMware - ${folderMap.name}"
                } else if (folderMap.name == "vm") {
                    return "VMs" // VMs and templates folder
                } else {
                    return folderMap.name
                }                
            } else if (type == "DATACENTER") {
                return "Datacenter - ${folderMap.name}"
            } else if (type == "CLUSTERCOMPUTERESOURCE") {
                return "Cluster - ${folderMap.name}"
            } else if (type == "RESOURCEPOOL") {
                return "Resource Pool - ${folderMap.name}"                
            } else if (type == "COMPUTERESOURCE") {
                return "Standalone ESXi hosts"
            } else {
                return folderMap.name
            }
            // Don't put the device in a folder named after it
        }[0..-2]

        if (lineage.size() >= 2) {
            // Check for standalone VMs that are not in a resourcePool, and put them in their own standalone folder
            if (lineage[-1].type == "VIRTUALMACHINE" && (lineage[-2].type != "RESOURCEPOOL" && lineage[-2].type != "FOLDER")) {
                if (includeStandaloneVM) {
                    folder << "Standalone VMs"
                } else {
                    // This is a standalone, and if we don't want to include it, return nothing
                    return null
                }
            }
        }
        return folder.join("/")
    }
    return null
}

Map lineageParser(lineage) {
    def out = [:].withDefault { [] }
    lineage.each { out[it?.type] << it?.name }
    out.collectEntries { k, v ->
        v.unique()
        if (k.toUpperCase() == "CLUSTERCOMPUTERESOURCE") {
            k = "CLUSTER"
        } else if (k.toUpperCase() == "FOLDER") {
            v.remove("vm") // Autogenerated folders
            v.remove("Datacenters")
        }
        _out = [k, v]
    }
}

/**
 * Sets the relationships map for the supplied Managed Object and all of it"s parents.
 * For most Managed Object types it only looks at the parent property, for virtual machines, the containing resourcePool is also mapped.
 * example for virtual machines this may map something like virtual machine -> resource pool -> host cluster -> folder -> datacenter
 * and virtual machine -> containing folders -> datacenter
 */
def updateEntityLineage(MO, Map relationships) {
    if (!MO) return
    int entityUpdateMax = 86400  // Time in seconds until cached data is forced to update, default 1 day
    int epoch = System.currentTimeSeconds()
    def mor = MO.MOR
    def entityID = mor?.val
    def type = mor?.type?.toUpperCase()
    def parent = MO.parent
    def parentID = parent?.MOR?.val
    def parentIDs = [parentID]
    if (parent && entityID) {
        int time = epoch - ((relationships.get(entityID)?.lastUpdate) ?: 0)
        if (time >= entityUpdateMax || !relationships.get(entityID)?.parent?.contains(parentID)) {
            // Now check parent
            updateEntityLineage(parent, relationships)
            if (type == "VIRTUALMACHINE") {
                // For VMs we want to both trace through the compute resources (resource pool) as well as the actual parent (folder)
                def rp = MO.resourcePool
                def rpID = rp?.MOR?.val
                parentIDs << rpID
                updateEntityLineage(rp, relationships)
            }
        }
    }
    def relationship = [
        "name"      : MO.name,
        "type"      : type,
        "class"     : MO.class?.getName(),
        "lastUpdate": epoch,
        "parent"    : parentIDs?.minus(null),
        "MO"        : MO
    ]
    relationships.put(entityID, relationship)
}

/**
 * Returns the supplied inventory item"s computational lineage as based on the relationships map.
 * This is based on the Hosts and Clusters Inventory Lists view in the vCenter UI. This view contains the virtual machine, resource pools, esx hosts, clusters, datacenters and more.
 */
def getLineage(relationshipType, entityID, Map relationships, lineage = []) {
    if( !entityID || !relationships )  return lineage
    def entry = relationships.get(entityID)
    if (entry) {
        // If the vm host is running vcsa 6.7 and is not in a resourcepool, then we can't currently trace its compute lineage
        def parent = entry.type == "VIRTUALMACHINE" ? entry.parent.find{ relationships.get(it)?.type == relationshipType } : entry.parent[0]
        if (entry.name != "host" && entry.name != "Resources") {
            lineage.add(0, entry)
        }
        getLineage(relationshipType, parent, relationships, lineage)
    }
    return lineage
}