From bc6d2e8a5f7bcb993906afb5df780b24ab1094e1 Mon Sep 17 00:00:00 2001 From: Brian Wuchner Date: Mon, 21 Feb 2022 12:21:45 -0500 Subject: [PATCH 1/6] Initial commit of VMware.Skyline.InsightsApi module Initial commit of VMware.SkylineInsightsApi module, containing the following functions: Connect-SkylineInsights, Disconnect-SkylineInsights, Invoke-SkylineInsightsApi, Get-SkylineFinding, Get-SkylineAffectedObject, Format-SkylineResult, Start-SkylineInsightsApiExplorer Signed-off-by: Brian Wuchner --- .../VMware.Skyline.InsightsApi.Format.ps1xml | 41 ++ .../VMware.Skyline.InsightsApi.psd1 | 128 ++++++ .../VMware.Skyline.InsightsApi.psm1 | 397 ++++++++++++++++++ 3 files changed, 566 insertions(+) create mode 100644 Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.Format.ps1xml create mode 100644 Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psd1 create mode 100644 Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 diff --git a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.Format.ps1xml b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.Format.ps1xml new file mode 100644 index 0000000..078204f --- /dev/null +++ b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.Format.ps1xml @@ -0,0 +1,41 @@ + + + + + SkylineConnection + + SkylineConnection + + + + + 30 + + + + 30 + + + + + + + + + + + Name + + + APIKey + + + CSPName + + + + + + + + \ No newline at end of file diff --git a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psd1 b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psd1 new file mode 100644 index 0000000..2eb6425 --- /dev/null +++ b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psd1 @@ -0,0 +1,128 @@ +<# +Copyright 2021 VMware, Inc. +SPDX-License-Identifier: BSD-2-Clause +#> + +# +# Module manifest for module 'VMware.Skyline.InsightsApi' +# +# Generated by: Brian Wuchner +# +# Generated on: 2/21/2022 +# + +@{ + +# Script module or binary module file associated with this manifest. +RootModule = 'VMware.Skyline.InsightsApi.psm1' + +# Version number of this module. +ModuleVersion = '1.0.0' + +# Supported PSEditions +# CompatiblePSEditions = @() + +# ID used to uniquely identify this module +GUID = '4dfcb1e5-69b9-405d-aecd-06119ec12649' + +# Author of this module +Author = 'Brian Wuchner' + +# Company or vendor of this module +CompanyName = 'VMware' + +# Copyright statement for this module +Copyright = '(c) VMware. All rights reserved.' + +# Description of the functionality provided by this module +Description = 'Community sourced PowerShell wrapper module for the Skyline Insights API.' + +# Minimum version of the Windows PowerShell engine required by this module +PowerShellVersion = '4.0' + +# Name of the Windows PowerShell host required by this module +# PowerShellHostName = '' + +# Minimum version of the Windows PowerShell host required by this module +# PowerShellHostVersion = '' + +# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# DotNetFrameworkVersion = '' + +# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# CLRVersion = '' + +# Processor architecture (None, X86, Amd64) required by this module +# ProcessorArchitecture = '' + +# Modules that must be imported into the global environment prior to importing this module +# RequiredModules = @() + +# Assemblies that must be loaded prior to importing this module +# RequiredAssemblies = @() + +# Script files (.ps1) that are run in the caller's environment prior to importing this module. +# ScriptsToProcess = @() + +# Type files (.ps1xml) to be loaded when importing this module +# TypesToProcess = @() + +# Format files (.ps1xml) to be loaded when importing this module +FormatsToProcess = @('VMware.Skyline.InsightsApi.Format.ps1xml') + +# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess +# NestedModules = @() + +# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. +FunctionsToExport = @('Connect-SkylineInsights','Disconnect-SkylineInsights','Invoke-SkylineInsightsApi','Get-SkylineFinding', + 'Get-SkylineAffectedObject','Format-SkylineResult','Start-SkylineInsightsApiExplorer') + +# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. +CmdletsToExport = @() + +# Variables to export from this module +VariablesToExport = '*' + +# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. +AliasesToExport = @() + +# DSC resources to export from this module +# DscResourcesToExport = @() + +# List of all modules packaged with this module +# ModuleList = @() + +# List of all files packaged with this module +# FileList = @() + +# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. +PrivateData = @{ + + PSData = @{ + + # Tags applied to this module. These help with module discovery in online galleries. + # Tags = @() + + # A URL to the license for this module. + # LicenseUri = '' + + # A URL to the main website for this project. + # ProjectUri = '' + + # A URL to an icon representing this module. + # IconUri = '' + + # ReleaseNotes of this module + # ReleaseNotes = '' + + } # End of PSData hashtable + +} # End of PrivateData hashtable + +# HelpInfo URI of this module +# HelpInfoURI = '' + +# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. +# DefaultCommandPrefix = '' + +} diff --git a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 new file mode 100644 index 0000000..012004e --- /dev/null +++ b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 @@ -0,0 +1,397 @@ +<# +Copyright 2021 VMware, Inc. +SPDX-License-Identifier: BSD-2-Clause +#> + +Function Connect-SkylineInsights { +<# + .NOTES + =========================================================================== + Created by: Brian Wuchner + Date: February 21, 2022 + Blog: www.enterpriseadmins.org + Twitter: @bwuch + =========================================================================== + .SYNOPSIS + Use this function to create the auth header to connect to Skyline Insights API + .DESCRIPTION + This function will allow you to connect to a Skyline Insights API. + A global variable will be set with the Servername & Header value for use by other functions. + .EXAMPLE + PS C:\> Connect-SkylineInsights -apiKey 'my-key-from-csp' + This will use the provided API key to create a connection to Skyline Insights. + .EXAMPLE + PS C:\> Connect-SkylineInsights -apiKey 'my-key-from-csp' -SaveCredentials + This will use the PowerCLI VICredentialStore Item to save the provided API key. On next use this key will be provided automatically. +#> + param( + [string]$apiKey, + [switch]$SaveCredentials, + [Parameter(DontShow)]$cspApi = 'console.cloud.vmware.com', + [Parameter(DontShow)]$skylineApi = 'skyline.vmware.com' + ) + + # Create VICredentialStore item to save the API key + if ($apiKey -AND $SaveCredentials) { + if ( (Get-Command Get-VICredentialStoreItem -ErrorAction:SilentlyContinue | Measure-Object).Count -gt 0 ) { + $savedCred = Get-VICredentialStoreItem -host $skylineApi -ErrorAction:SilentlyContinue + if ($savedCred) { + $savedCred | Remove-VICredentialStoreItem -Confirm:$false + } + New-VICredentialStoreItem -Host $skylineApi -User 'api-key' -Password $apiKey + } else { + Write-Warning 'Use of -SaveCredentials requires the PowerCLI VICredentialStoreItem cmdlets.' + } + } + + if (!$apiKey) { + if ( (Get-Command Get-VICredentialStoreItem -ErrorAction:SilentlyContinue | Measure-Object).Count -gt 0 ) { + $savedCred = Get-VICredentialStoreItem -host $skylineApi -ErrorAction:SilentlyContinue + } + if ( ($savedCred | Measure-Object).Count -eq 1) { + $apiKey = $savedCred.Password + } else { + write-error 'An API key is required.' + return + } + } + + $loginHeader = @{ + 'Accept' = 'application/json' + 'Content-Type' = 'application/x-www-form-urlencoded' + } + $loginBody = @{'refresh_token' = $apiKey } + + try { + $webRequest = Invoke-RestMethod -Uri "https://$cspApi/csp/gateway/am/api/auth/api-tokens/authorize?grant_type=refresh_token" -method POST -Headers $loginHeader -Body $loginBody + + $global:DefaultSkylineConnection = New-Object psobject -property @{ 'Name'=$skylineApi; 'CSPName'=$cspApi; 'ConnectionDetail'=$webRequest; APIKey = $apiKey; + 'Refresh_Token'=$webRequest.refresh_token; 'SkylineAPI'="https://$skylineApi/public/api/data"; PSTypeName='SkylineConnection' } + + # Return the connection object + $global:DefaultSkylineConnection + } catch { + Write-Error ("Failure connecting to $skylineAPI. Posted $loginBody " + $_) + } # end try/catch block +} + +Function Disconnect-SkylineInsights { +<# + .NOTES + =========================================================================== + Created by: Brian Wuchner + Date: February 21, 2022 + Blog: www.enterpriseadmins.org + Twitter: @bwuch + =========================================================================== + .SYNOPSIS + Use this function to disconnect from Skyline Insights API + .DESCRIPTION + This function will allow you to disconnect from a Skyline Insights API. + The global variable will be set with the Servername & Header value for use by other functions. + .EXAMPLE + PS C:\> Disconnect-SkylineInsights + This will remove a connection to Skyline Insights. +#> + if ($global:DefaultSkylineConnection) { + $global:DefaultSkylineConnection = $null + } else { + Write-Error 'Could not find an existing connection to SkylineInsights API.' + } +} + +Function Invoke-SkylineInsightsApi { +<# + .NOTES + =========================================================================== + Created by: Brian Wuchner + Date: February 21, 2022 + Blog: www.enterpriseadmins.org + Twitter: @bwuch + =========================================================================== + .SYNOPSIS + Use this function to post a query to the Skyline Insights API. + .DESCRIPTION + This function will allow you to query the Skyline Insights API. + Proper headers will be formatted and posted if a DefaultSkylineConnection is present. + This is primarily a helper function used by other functions included in the module. + It is exported in the module manifest to be used for any custom queries. + .EXAMPLE + PS C:\> Invoke-SkylineInsightsApi -queryBody '{formatted-query-string-converted-to-json}' +#> + param( + [Parameter(Mandatory=$true)][string]$queryBody + ) + + if ( !$global:DefaultSkylineConnection ) { + Write-Error 'You are not currently connected to any servers. Please connect first using Connect-SkylineInsights.' + return; + } + + write-debug "Querybody: $queryBody" + try { + $restCall = invoke-restmethod -method post -Uri $($global:DefaultSkylineConnection.SkylineAPI) -Headers @{Authorization = "Bearer $($global:DefaultSkylineConnection.ConnectionDetail.access_token)"} -body $queryBody -ContentType "application/json" + if ($restCall.errors) { + Write-Error $restCall.errors.Message + } + return $restCall + } catch { + $incomingError = $_ + try { + # are nested try/catch blocks the powershell equilivent of vbscript On Error Resume Next? + $errorStatusAsJson = ($incomingError | ConvertFrom-Json).status + if ($errorStatusAsJson -eq '429 TOO_MANY_REQUESTS') { + write-debug 'Hit 429 / rate limite warning, will sleep for 1 second.' + start-sleep -Milliseconds 1005 + Invoke-SkylineInsightsApi $queryBody + } + } catch { + # this was the error from trying to cast the incoming error to Json + } + if (!$errorStatusAsJson) { write-error $_ } + } +} + + +Function Get-SkylineFinding { +<# + .NOTES + =========================================================================== + Created by: Brian Wuchner + Date: February 21, 2022 + Blog: www.enterpriseadmins.org + Twitter: @bwuch + =========================================================================== + .SYNOPSIS + Use this function to query findings from the Skyline Insights API. + .DESCRIPTION + This function will allow you to query the Skyline Insights API for Findings. + As described in the documentation, the maximum limit per page is 200 records. This function provides + an optional pagesize parameter to request smaller batches, but by default assumes 200 records. + .EXAMPLE + PS C:\> Get-SkylineFinding +#> + [cmdletbinding()] + param( + [ValidateRange(1,200)][int]$pagesize=200 + ) + $queryBody = @" +{ + activeFindings(limit: $pagesize, start: 0 ) { + findings { + findingId + accountId + findingDisplayName + severity + products + findingDescription + findingImpact + recommendations + kbLinkURLs + recommendationsVCF + kbLinkURLsVCF + categoryName + findingTypes + firstObserved + totalAffectedObjectsCount + } + totalRecords + timeTaken + } +} +"@ + + # Try to get results the first time + $results = @() + $thisQueryBody = $queryBody + $thisIteration = 0 + do { + $thisQueryBody = $thisQueryBody -Replace 'start: 0', "start: $thisIteration" + Write-Debug $thisQueryBody + $thisResult = Invoke-SkylineInsightsApi -queryBody (@{'query' = $thisQueryBody} | ConvertTo-Json -Compress) + $totalRecords = $thisResult.data.activeFindings.totalRecords + $results += ($thisResult.data.activeFindings.Findings) + $thisIteration += $pageSize + } while ($results.count -lt $totalRecords ) # end do/while loop + + return $results +} + +Function Get-SkylineAffectedObject { +<# + .NOTES + =========================================================================== + Created by: Brian Wuchner + Date: February 21, 2022 + Blog: www.enterpriseadmins.org + Twitter: @bwuch + =========================================================================== + .SYNOPSIS + Use this function to query affected objects from the Skyline Insights API. + .DESCRIPTION + This function will allow you to query the Skyline Insights API for affected objects. + Input parameters are required for the findingId and product. Products can be provided as an object (from Get-SkylineFinding) or + a single product can be specified by name (or delimited list). + As described in the documentation, the maximum limit per page is 200 records. This function provides + an optional pagesize parameter to request smaller batches, but by default assumes 200 records. + .EXAMPLE + PS C:\> Get-SkylineAffectedObject -findingId 'vSphere-Vmtoolsmemoryleak-KB#76163' -product 'core-vcenter01.lab.enterpriseadmins.org' + This example uses the ByName parameter set to pass in specific findings/product and expects either a single product or a 'separator' delimited list + .EXAMPLE + PS C:\> Get-SkylineFinding | Select-Object -First 2 | Get-SkylineAffectedObject + This example uses the ByObject parameter set to pass in products as an object from Get-SkylineFinding +#> + [cmdletbinding()] + param( + [Parameter(Mandatory=$true, ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)][string]$findingId, + [Parameter(Mandatory=$true, ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true,ParameterSetName='ByObject')][System.Object]$products, + [Parameter(Mandatory=$true, ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true,ParameterSetName='ByName')][string]$product, + [Parameter(ParameterSetName='ByName')][string]$separator = '; ', + [ValidateRange(1,200)][int]$pagesize=200 + ) + + begin { + $queryBody = @" + { + activeFindings( + filter: { + findingId: "$findingId", + product: "", + }) { + findings { + totalAffectedObjectsCount + affectedObjects(start: 0, limit: $pagesize) { + sourceName + objectName + objectType + version + buildNumber + solutionTags { + type + version + } + firstObserved + } + } + totalRecords + timeTaken + } + } +"@ + + # Try to get results the first time + $results = @() + + if ($PSCmdlet.ParameterSetName -eq 'ByName') { + $procProduct = $product.Split($separator).Trim() | Where-Object {$_ -ne '' } + # depending on separator we could end up with null rows on split, we'll omit those with where object + # to eliminate possible errors from the API + } + + if ($PSCmdlet.ParameterSetName -eq 'ByObject') { + $procProduct = $products.products + } + } + + process { + foreach ( $thisProduct in $procProduct ) { + $thisQueryBody = $queryBody -Replace 'findingId: "",', "findingId: `"$findingId`"," + $thisQueryBody = $thisQueryBody -Replace 'product: "",', "product: `"$thisProduct`"," + $thisIteration = 0 + + do { + $thisQueryBody = $thisQueryBody -Replace 'start: 0', "start: $thisIteration" + Write-Debug $thisQueryBody + $thisResult = Invoke-SkylineInsightsApi -queryBody (@{'query' = $thisQueryBody} | ConvertTo-Json -Compress) + $totalRecords = $thisResult.data.activeFindings.Findings.totalAffectedObjectsCount + $results += ($thisResult.data.activeFindings.Findings.affectedObjects) | Select-Object @{N='findingId';E={$findingId}}, * + $thisIteration += $pagesize + } while ($results.count -lt $totalRecords ) # end do/while loop + } # end foreach product loop + } + + end { + return $results + } +} + +Function Format-SkylineResult { +<# + .NOTES + =========================================================================== + Created by: Brian Wuchner + Date: February 21, 2022 + Blog: www.enterpriseadmins.org + Twitter: @bwuch + =========================================================================== + .SYNOPSIS + Use this function to format results from the Skyline Insights API + .DESCRIPTION + This function will format the output from the Skyline Insights API. + For example, Get-SkylineFinding and Get-SkylineAffectedObject will return some strings, date values as numbers, and object properties. + This function will convert date numbers to powershell dates and objects to delimiter separated stings. This should help with exporting + results to CSV files for example. + .EXAMPLE + PS C:\> Get-SkylineFinding | Format-SkylineResult | Export-Csv c:\temp\findings.csv -NoTypeInformation + This will return Skyline Findings, format them as needed, and export results to a CSV file. +#> + param( + [Parameter(Mandatory=$true, ValueFromPipeline=$true)][PSCustomObject]$inputObject, + [string]$separator = '; ' + ) + begin { + $results = @() + + # To format the dates, we need to add the value returned by the API to the begining of time + $startOfTime = Get-Date '1970-01-01' + } + + process { + if ( $inputObject.accountId ) { + #This appears to be a Finding + $results += $inputObject | Select-Object findingId, accountId, findingDisplayName, severity, @{N='product';E={[string]::join($separator, $_.products)}}, findingDescription, + findingImpact, @{N='recommendations';E={[string]::Join($separator,$_.recommendations)}}, @{N='kbLinkURLs';E={[string]::Join($separator, $_.kbLinkURLs)}}, + @{N='recommendationsVCF';E={[string]::Join($separator,$_.recommendationsVCF)}}, @{N='kbLinkURLsVCF';E={[string]::Join($separator, $_.kbLinkURLsVCF)}}, + categoryName, @{N='findingTypes';E={[string]::Join($sep, $_.findingTypes)}}, @{N='firstObserved';E={ $startOfTime+[timespan]::FromMilliseconds($_.firstObserved) }}, + totalAffectedObjectsCount + + } elseif ( $inputObject.objectName ) { + #This appears to be an AffectedObject + $results += $inputObject | Select-Object findingId, sourceName, objectName, objectType, version, buildNumber, @{N='solutionTags-Type';E={$_.solutionTags.type}}, + @{N='solutionTags-Version';E={$_.solutionTags.version}}, @{N='firstObserved';E={ $startOfTime+[timespan]::FromMilliseconds($_.firstObserved) }} + } else { + write-warning "Unable to determine input object type." + } # end inputobject evaluation + } #end process + + end { + return $results + } +} + +Function Start-SkylineInsightsApiExplorer { +<# + .NOTES + =========================================================================== + Created by: Brian Wuchner + Date: February 21, 2022 + Blog: www.enterpriseadmins.org + Twitter: @bwuch + =========================================================================== + .SYNOPSIS + Use this function to launch the Skyline Insights API in a browser. + .DESCRIPTION + This function will open the Skyline Insights API explorer in the default web browser and populate + the clipboard with the necessary authorization header value to enable interactive queries. + .EXAMPLE + PS C:\> Start-SkylineInsightsApiExplorer +#> + if ( !$global:DefaultSkylineConnection ) { + Write-Error 'You are not currently connected to any servers. Please connect first using Connect-SkylineInsights.' + return; + } + "Default web browser will launch to the Skyline Insights API explorer. In the lower left select 'Request Headers' and paste the authorization/bearer token into the text box. `nNote: this script has updated your clipboard with the required auth token." + "{`"Authorization`":`"Bearer $($global:DefaultSkylineConnection.ConnectionDetail.access_token)`"}" | Set-Clipboard + Start-Process "https://$($global:DefaultSkylineConnection.Name)/public/api/docs" +} + From 3cd0fe0ca590329ce4c9b304ea9e8e4201bd40f7 Mon Sep 17 00:00:00 2001 From: Brian Wuchner Date: Wed, 23 Feb 2022 15:18:04 -0500 Subject: [PATCH 2/6] Update VMware.Skyline.InsightsApi.psm1 Moving the ParameterSetName checks to the process block instead of the begin block. Signed-off-by: Brian Wuchner --- .../VMware.Skyline.InsightsApi.psm1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 index 012004e..a210389 100644 --- a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 +++ b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 @@ -281,7 +281,9 @@ Function Get-SkylineAffectedObject { # Try to get results the first time $results = @() + } + process { if ($PSCmdlet.ParameterSetName -eq 'ByName') { $procProduct = $product.Split($separator).Trim() | Where-Object {$_ -ne '' } # depending on separator we could end up with null rows on split, we'll omit those with where object @@ -291,9 +293,7 @@ Function Get-SkylineAffectedObject { if ($PSCmdlet.ParameterSetName -eq 'ByObject') { $procProduct = $products.products } - } - - process { + foreach ( $thisProduct in $procProduct ) { $thisQueryBody = $queryBody -Replace 'findingId: "",', "findingId: `"$findingId`"," $thisQueryBody = $thisQueryBody -Replace 'product: "",', "product: `"$thisProduct`"," From d85c6096a317e68b9940d00d5135b3a19dc30a63 Mon Sep 17 00:00:00 2001 From: Brian Wuchner Date: Fri, 18 Mar 2022 23:04:05 -0400 Subject: [PATCH 3/6] Update VMware.Skyline.InsightsApi.psm1 Addressing issues in PR 546. Signed-off-by: Brian Wuchner --- .../VMware.Skyline.InsightsApi.psm1 | 94 +++++++++++-------- 1 file changed, 56 insertions(+), 38 deletions(-) diff --git a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 index a210389..bd2c4db 100644 --- a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 +++ b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 @@ -31,6 +31,16 @@ Function Connect-SkylineInsights { [Parameter(DontShow)]$skylineApi = 'skyline.vmware.com' ) + if ($PSEdition -eq 'Core' -And $SaveCredentials) { + write-error 'The parameter SaveCredentials of Connect-SkylineInsights cmdlet is not supported on PowerShell Core.' + return + } + + if ($PSEdition -eq 'Core' -AND !$apiKey) { + write-error 'An API key is required.' + return + } + # Create VICredentialStore item to save the API key if ($apiKey -AND $SaveCredentials) { if ( (Get-Command Get-VICredentialStoreItem -ErrorAction:SilentlyContinue | Measure-Object).Count -gt 0 ) { @@ -143,7 +153,8 @@ Function Invoke-SkylineInsightsApi { if ($errorStatusAsJson -eq '429 TOO_MANY_REQUESTS') { write-debug 'Hit 429 / rate limite warning, will sleep for 1 second.' start-sleep -Milliseconds 1005 - Invoke-SkylineInsightsApi $queryBody + $retryQuery = Invoke-SkylineInsightsApi $queryBody + return $retryQuery } } catch { # this was the error from trying to cast the incoming error to Json @@ -173,11 +184,16 @@ Function Get-SkylineFinding { #> [cmdletbinding()] param( - [ValidateRange(1,200)][int]$pagesize=200 + [Parameter(ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)][string]$findingId, + [Parameter(ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)][string[]]$products, + [Parameter(ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)][ValidateSet('CRITICAL','MODERATE','TRIVIAL')][string]$severity, + [Parameter(DontShow=$true)][ValidateRange(1,200)][int]$pagesize=200 ) - $queryBody = @" + + begin { + $queryBody = @" { - activeFindings(limit: $pagesize, start: 0 ) { + activeFindings(limit: $pagesize, start: 0 filter: {}) { findings { findingId accountId @@ -201,20 +217,37 @@ Function Get-SkylineFinding { } "@ - # Try to get results the first time - $results = @() - $thisQueryBody = $queryBody - $thisIteration = 0 - do { - $thisQueryBody = $thisQueryBody -Replace 'start: 0', "start: $thisIteration" - Write-Debug $thisQueryBody - $thisResult = Invoke-SkylineInsightsApi -queryBody (@{'query' = $thisQueryBody} | ConvertTo-Json -Compress) - $totalRecords = $thisResult.data.activeFindings.totalRecords - $results += ($thisResult.data.activeFindings.Findings) - $thisIteration += $pageSize - } while ($results.count -lt $totalRecords ) # end do/while loop + } + process { + if (!$products) { $products = 'NO_PRODUCT_FILTER'} + foreach ($thisProduct in $products) { + if ($findingId) { $filterString = "findingId: `"$findingId`"," } + if ($thisProduct -ne 'NO_PRODUCT_FILTER') { $filterString += "product: `"$thisProduct`"," } - return $results + # Try to get results the first time + $results = @() + $thisQueryBody = $queryBody -Replace 'filter: {}', "filter: { $filterString }" + $thisIteration = 0 + do { + $thisQueryBody = $thisQueryBody -Replace 'start: 0', "start: $thisIteration" + Write-Debug $thisQueryBody + $thisResult = Invoke-SkylineInsightsApi -queryBody (@{'query' = $thisQueryBody} | ConvertTo-Json -Compress) + $totalRecords = $thisResult.data.activeFindings.totalRecords + if ($severity) { + $thisResult.data.activeFindings.Findings | Where-Object {$_.severity -eq $severity} + } else { + $thisResult.data.activeFindings.Findings + } + $results += ($thisResult.data.activeFindings.Findings) + $thisIteration += $pageSize + } while ($results.count -lt $totalRecords ) # end do/while loop + + #return $results + } + } + end { + + } } Function Get-SkylineAffectedObject { @@ -244,10 +277,8 @@ Function Get-SkylineAffectedObject { [cmdletbinding()] param( [Parameter(Mandatory=$true, ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)][string]$findingId, - [Parameter(Mandatory=$true, ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true,ParameterSetName='ByObject')][System.Object]$products, - [Parameter(Mandatory=$true, ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true,ParameterSetName='ByName')][string]$product, - [Parameter(ParameterSetName='ByName')][string]$separator = '; ', - [ValidateRange(1,200)][int]$pagesize=200 + [Parameter(Mandatory=$true, ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)][string[]]$products, + [Parameter(DontShow=$true)][ValidateRange(1,200)][int]$pagesize=200 ) begin { @@ -255,8 +286,8 @@ Function Get-SkylineAffectedObject { { activeFindings( filter: { - findingId: "$findingId", - product: "", + findingId: "$findingId", + product: "", }) { findings { totalAffectedObjectsCount @@ -284,17 +315,7 @@ Function Get-SkylineAffectedObject { } process { - if ($PSCmdlet.ParameterSetName -eq 'ByName') { - $procProduct = $product.Split($separator).Trim() | Where-Object {$_ -ne '' } - # depending on separator we could end up with null rows on split, we'll omit those with where object - # to eliminate possible errors from the API - } - - if ($PSCmdlet.ParameterSetName -eq 'ByObject') { - $procProduct = $products.products - } - - foreach ( $thisProduct in $procProduct ) { + foreach ( $thisProduct in $products ) { $thisQueryBody = $queryBody -Replace 'findingId: "",', "findingId: `"$findingId`"," $thisQueryBody = $thisQueryBody -Replace 'product: "",', "product: `"$thisProduct`"," $thisIteration = 0 @@ -304,15 +325,12 @@ Function Get-SkylineAffectedObject { Write-Debug $thisQueryBody $thisResult = Invoke-SkylineInsightsApi -queryBody (@{'query' = $thisQueryBody} | ConvertTo-Json -Compress) $totalRecords = $thisResult.data.activeFindings.Findings.totalAffectedObjectsCount + $thisResult.data.activeFindings.Findings.affectedObjects | Select-Object @{N='findingId';E={$findingId}}, * $results += ($thisResult.data.activeFindings.Findings.affectedObjects) | Select-Object @{N='findingId';E={$findingId}}, * $thisIteration += $pagesize } while ($results.count -lt $totalRecords ) # end do/while loop } # end foreach product loop } - - end { - return $results - } } Function Format-SkylineResult { From b2e0decb6844ab3db917b5b89419acbb79b6ac97 Mon Sep 17 00:00:00 2001 From: Brian Wuchner Date: Sun, 20 Mar 2022 20:50:13 -0400 Subject: [PATCH 4/6] Update VMware.Skyline.InsightsApi.psm1 Fixing minor issue uncovered by testing where only a product (vCenter Name) was passed to Get-SkylineFinding function. When passed by pipeline, the product was applied to each pipeline input. Making a change to require passing through pipeline by property name only. Signed-off-by: Brian Wuchner --- .../VMware.Skyline.InsightsApi.psm1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 index bd2c4db..a4d2a8e 100644 --- a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 +++ b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 @@ -184,9 +184,9 @@ Function Get-SkylineFinding { #> [cmdletbinding()] param( - [Parameter(ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)][string]$findingId, - [Parameter(ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)][string[]]$products, - [Parameter(ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)][ValidateSet('CRITICAL','MODERATE','TRIVIAL')][string]$severity, + [Parameter(ValueFromPipelineByPropertyName=$true)][string]$findingId, + [Parameter(ValueFromPipelineByPropertyName=$true)][string[]]$products, + [Parameter(ValueFromPipelineByPropertyName=$true)][ValidateSet('CRITICAL','MODERATE','TRIVIAL')][string]$severity, [Parameter(DontShow=$true)][ValidateRange(1,200)][int]$pagesize=200 ) From 80622414d6c5e6fad0119df5246ca74264abb0e8 Mon Sep 17 00:00:00 2001 From: Brian Wuchner Date: Fri, 13 May 2022 11:14:46 -0400 Subject: [PATCH 5/6] Update VMware.Skyline.InsightsApi.psm1 Apologies for the delay on getting this commit to address the final open item in the PR. This change removes the "hit a 429 and retry" logic as it was not effective. Replaced it with a global variable that stores the time of the last query. If a query has happened within the last 501ms we wait before sending. I've issued a few thousand queries with this logic added and have not yet hit the 429 error. The logic to find and report on 429's in a more friendly way still exists, just in case. Additionally I've implemented a counter to track number of queries that is reset by Connect-SkylineInsights, to track how many queries are executed. This was more of a debugging tool, but felt the overhead was low enough to leave it in for future troubleshooting. Signed-off-by: Brian Wuchner --- .../VMware.Skyline.InsightsApi.psm1 | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 index a4d2a8e..217ffb4 100644 --- a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 +++ b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 @@ -79,6 +79,8 @@ Function Connect-SkylineInsights { 'Refresh_Token'=$webRequest.refresh_token; 'SkylineAPI'="https://$skylineApi/public/api/data"; PSTypeName='SkylineConnection' } # Return the connection object + $global:SkylineInsightsApiQueryCount = 0 + $global:SkylineInsightsApiQueryLastTime = $null $global:DefaultSkylineConnection } catch { Write-Error ("Failure connecting to $skylineAPI. Posted $loginBody " + $_) @@ -130,7 +132,8 @@ Function Invoke-SkylineInsightsApi { PS C:\> Invoke-SkylineInsightsApi -queryBody '{formatted-query-string-converted-to-json}' #> param( - [Parameter(Mandatory=$true)][string]$queryBody + [Parameter(Mandatory=$true)][string]$queryBody, + [Parameter(DontShow=$true)][int]$sleepTimerMs=501 ) if ( !$global:DefaultSkylineConnection ) { @@ -140,7 +143,16 @@ Function Invoke-SkylineInsightsApi { write-debug "Querybody: $queryBody" try { + if ($global:SkylineInsightsApiQueryLastTime) { + $timeSinceLastQuery = (New-TimeSpan $global:SkylineInsightsApiQueryLastTime (Get-Date)).TotalMilliseconds + if ($timeSinceLastQuery -lt $sleepTimerMs) { + Write-Debug "Waiting $($sleepTimerMs-$timeSinceLastQuery)ms to prevent HTTP 429 TOO_MANY_REQUESTS error" + Start-Sleep -Milliseconds ($sleepTimerMs-$timeSinceLastQuery) + } + } $restCall = invoke-restmethod -method post -Uri $($global:DefaultSkylineConnection.SkylineAPI) -Headers @{Authorization = "Bearer $($global:DefaultSkylineConnection.ConnectionDetail.access_token)"} -body $queryBody -ContentType "application/json" + $global:SkylineInsightsApiQueryCount++ + $global:SkylineInsightsApiQueryLastTime = Get-Date if ($restCall.errors) { Write-Error $restCall.errors.Message } @@ -151,15 +163,14 @@ Function Invoke-SkylineInsightsApi { # are nested try/catch blocks the powershell equilivent of vbscript On Error Resume Next? $errorStatusAsJson = ($incomingError | ConvertFrom-Json).status if ($errorStatusAsJson -eq '429 TOO_MANY_REQUESTS') { - write-debug 'Hit 429 / rate limite warning, will sleep for 1 second.' - start-sleep -Milliseconds 1005 - $retryQuery = Invoke-SkylineInsightsApi $queryBody - return $retryQuery + write-error 'Encountered HTTP 429 TOO_MANY_REQUESTS error, consider increasing sleepTimerMs value.' + start-sleep -Milliseconds (2*$sleepTimerMs) + break } } catch { # this was the error from trying to cast the incoming error to Json } - if (!$errorStatusAsJson) { write-error $_ } + if (!$errorStatusAsJson) { write-error $incomingError } } } @@ -316,11 +327,10 @@ Function Get-SkylineAffectedObject { process { foreach ( $thisProduct in $products ) { - $thisQueryBody = $queryBody -Replace 'findingId: "",', "findingId: `"$findingId`"," - $thisQueryBody = $thisQueryBody -Replace 'product: "",', "product: `"$thisProduct`"," $thisIteration = 0 - + $results = @() # reset results variable between products do { + $thisQueryBody = $queryBody -Replace 'product: "",', "product: `"$thisProduct`"," $thisQueryBody = $thisQueryBody -Replace 'start: 0', "start: $thisIteration" Write-Debug $thisQueryBody $thisResult = Invoke-SkylineInsightsApi -queryBody (@{'query' = $thisQueryBody} | ConvertTo-Json -Compress) From e0e2d1bbdc5836dd63e5a8aaa7a418a1e30ec2f2 Mon Sep 17 00:00:00 2001 From: Brian Wuchner Date: Fri, 13 May 2022 16:43:07 -0400 Subject: [PATCH 6/6] Update VMware.Skyline.InsightsApi.psm1 I accidentally committed a version with a query problem in Get-SkylineAffectedObject. This version does not contain that search problem. Signed-off-by: Brian Wuchner --- .../VMware.Skyline.InsightsApi.psm1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 index 217ffb4..8349088 100644 --- a/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 +++ b/Modules/VMware.Skyline.InsightsApi/VMware.Skyline.InsightsApi.psm1 @@ -297,7 +297,7 @@ Function Get-SkylineAffectedObject { { activeFindings( filter: { - findingId: "$findingId", + findingId: "", product: "", }) { findings { @@ -322,15 +322,15 @@ Function Get-SkylineAffectedObject { "@ # Try to get results the first time - $results = @() } process { + $thisQueryBody = $queryBody -Replace 'findingId: "",', "findingId: `"$findingId`"," foreach ( $thisProduct in $products ) { $thisIteration = 0 $results = @() # reset results variable between products do { - $thisQueryBody = $queryBody -Replace 'product: "",', "product: `"$thisProduct`"," + $thisQueryBody = $thisQueryBody -Replace 'product: "",', "product: `"$thisProduct`"," $thisQueryBody = $thisQueryBody -Replace 'start: 0', "start: $thisIteration" Write-Debug $thisQueryBody $thisResult = Invoke-SkylineInsightsApi -queryBody (@{'query' = $thisQueryBody} | ConvertTo-Json -Compress)