Skip to content

Commit 3a5302c

Browse files
princepereiraprincepereira
committed
Network health script.
1 parent 7045e21 commit 3a5302c

2 files changed

Lines changed: 364 additions & 0 deletions

File tree

‎scripts/networkhealth/checkPortExhaustion.ps1‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
function CountAvailableEphemeralPorts([string]$protocol = "TCP") {
2+
3+
[uint32]$portRangeSize = 64
4+
# First, remove all the text bells and whistle (plain text, table headers, dashes, empty lines, ...) from netsh output
5+
$tcpRanges = (netsh int ipv4 sh excludedportrange $protocol) -replace "[^0-9,\ ]", '' | ? { $_.trim() -ne "" }
6+
7+
# Then, remove any extra space characters. Only capture the numbers representing the beginning and end of range
8+
$tcpRangesArray = $tcpRanges -replace "\s+(\d+)\s+(\d+)\s+", '$1,$2' | ConvertFrom-String -Delimiter ","
9+
#Convert from PSCustomObject to Object[] type
10+
$tcpRangesArray = @($tcpRangesArray)
11+
12+
# Extract the ephemeral ports ranges
13+
$EphemeralPortRange = (netsh int ipv4 sh dynamicportrange $protocol) -replace "[^0-9]", '' | ? { $_.trim() -ne "" }
14+
$EphemeralPortStart = [Convert]::ToUInt32($EphemeralPortRange[0])
15+
$EphemeralPortEnd = $EphemeralPortStart + [Convert]::ToUInt32($EphemeralPortRange[1]) - 1
16+
17+
# Find the external interface
18+
$externalInterfaceIdx = (Get-NetRoute -DestinationPrefix "0.0.0.0/0")[0].InterfaceIndex
19+
$hostIP = (Get-NetIPConfiguration -ifIndex $externalInterfaceIdx).IPv4Address.IPAddress
20+
21+
# Extract the used TCP ports from the external interface
22+
$usedTcpPorts = (Get-NetTCPConnection -LocalAddress $hostIP -ErrorAction Ignore).LocalPort
23+
$usedTcpPorts | % { $tcpRangesArray += [pscustomobject]@{P1 = $_; P2 = $_ } }
24+
25+
# Extract the used TCP ports from the 0.0.0.0 interface
26+
$usedTcpGlobalPorts = (Get-NetTCPConnection -LocalAddress "0.0.0.0" -ErrorAction Ignore).LocalPort
27+
$usedTcpGlobalPorts | % { $tcpRangesArray += [pscustomobject]@{P1 = $_; P2 = $_ } }
28+
# Sort the list and remove duplicates
29+
$tcpRangesArray = ($tcpRangesArray | Sort-Object { $_.P1 } -Unique)
30+
31+
$tcpRangesList = New-Object System.Collections.ArrayList($null)
32+
$tcpRangesList.AddRange($tcpRangesArray)
33+
34+
# Remove overlapping ranges
35+
for ($i = $tcpRangesList.P1.Length - 2; $i -gt 0 ; $i--) {
36+
if ($tcpRangesList[$i].P2 -gt $tcpRangesList[$i + 1].P1 ) {
37+
$tcpRangesList.Remove($tcpRangesList[$i + 1])
38+
$i++
39+
}
40+
}
41+
42+
# Remove the non-ephemeral port reservations from the list
43+
$filteredTcpRangeArray = $tcpRangesList | ? { $_.P1 -ge $EphemeralPortStart }
44+
$filteredTcpRangeArray = $filteredTcpRangeArray | ? { $_.P2 -le $EphemeralPortEnd }
45+
46+
if ($null -eq $filteredTcpRangeArray) {
47+
$freeRanges = @($EphemeralPortRange[1])
48+
}
49+
else {
50+
$freeRanges = @()
51+
# The first free range goes from $EphemeralPortStart to the beginning of the first reserved range
52+
$freeRanges += ([Convert]::ToUInt32($filteredTcpRangeArray[0].P1) - $EphemeralPortStart)
53+
54+
for ($i = 1; $i -lt $filteredTcpRangeArray.length; $i++) {
55+
# Subsequent free ranges go from the end of the previous reserved range to the beginning of the current reserved range
56+
$freeRanges += ([Convert]::ToUInt32($filteredTcpRangeArray[$i].P1) - [Convert]::ToUInt32($filteredTcpRangeArray[$i - 1].P2) - 1)
57+
}
58+
59+
# The last free range goes from the end of the last reserved range to $EphemeralPortEnd
60+
$freeRanges += ($EphemeralPortEnd - [Convert]::ToUInt32($filteredTcpRangeArray[$filteredTcpRangeArray.length - 1].P2))
61+
}
62+
63+
# Count the number of available free ranges
64+
[uint32]$freeRangesCount = 0
65+
($freeRanges | % { $freeRangesCount += [Math]::Floor($_ / $portRangeSize) } )
66+
67+
return $freeRangesCount
68+
}
69+
70+
function CheckPortExhaustion {
71+
Write-Host "Checking Port Exhaustion"
72+
$avTcpPorts = CountAvailableEphemeralPorts -protocol TCP
73+
if($avTcpPorts -lt 10) {
74+
Write-Host "Available TCP ports are $avTcpPorts. Port exhaustion suspected." -ForegroundColor Red
75+
return $true
76+
}
77+
$avUdpPorts = CountAvailableEphemeralPorts -protocol UDP
78+
if($avTcpPorts -lt 10) {
79+
Write-Host "Available UDP ports are $avUdpPorts. Port exhaustion suspected." -ForegroundColor Red
80+
return $true
81+
}
82+
Write-Host "Available TCP Ports : $avTcpPorts , UDP Ports : $avUdpPorts . No port exhaustion suspected." -ForegroundColor Green
83+
return $false
84+
}
85+
86+
Write-Host "Total wait time : 100 seconds..."
87+
88+
for($i = 1; $i -le 10; $i++) {
89+
Write-Host "Iteration : $i"
90+
if(CheckPortExhaustion) {
91+
Write-Host "DNS Issue Found." -ForegroundColor Red
92+
return
93+
}
94+
Start-Sleep -Seconds 10
95+
}

‎scripts/networkhealth/startNetworkDiagnostics.ps1‎

Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
param (
2+
[Parameter(Mandatory=$false)]
3+
[bool]$DnsPktCap = $false
4+
)
5+
6+
function CountAvailableEphemeralPorts([string]$protocol = "TCP") {
7+
8+
[uint32]$portRangeSize = 64
9+
# First, remove all the text bells and whistle (plain text, table headers, dashes, empty lines, ...) from netsh output
10+
$tcpRanges = (netsh int ipv4 sh excludedportrange $protocol) -replace "[^0-9,\ ]", '' | ? { $_.trim() -ne "" }
11+
12+
# Then, remove any extra space characters. Only capture the numbers representing the beginning and end of range
13+
$tcpRangesArray = $tcpRanges -replace "\s+(\d+)\s+(\d+)\s+", '$1,$2' | ConvertFrom-String -Delimiter ","
14+
#Convert from PSCustomObject to Object[] type
15+
$tcpRangesArray = @($tcpRangesArray)
16+
17+
# Extract the ephemeral ports ranges
18+
$EphemeralPortRange = (netsh int ipv4 sh dynamicportrange $protocol) -replace "[^0-9]", '' | ? { $_.trim() -ne "" }
19+
$EphemeralPortStart = [Convert]::ToUInt32($EphemeralPortRange[0])
20+
$EphemeralPortEnd = $EphemeralPortStart + [Convert]::ToUInt32($EphemeralPortRange[1]) - 1
21+
22+
# Find the external interface
23+
$externalInterfaceIdx = (Get-NetRoute -DestinationPrefix "0.0.0.0/0")[0].InterfaceIndex
24+
$hostIP = (Get-NetIPConfiguration -ifIndex $externalInterfaceIdx).IPv4Address.IPAddress
25+
26+
# Extract the used TCP ports from the external interface
27+
$usedTcpPorts = (Get-NetTCPConnection -LocalAddress $hostIP -ErrorAction Ignore).LocalPort
28+
$usedTcpPorts | % { $tcpRangesArray += [pscustomobject]@{P1 = $_; P2 = $_ } }
29+
30+
# Extract the used TCP ports from the 0.0.0.0 interface
31+
$usedTcpGlobalPorts = (Get-NetTCPConnection -LocalAddress "0.0.0.0" -ErrorAction Ignore).LocalPort
32+
$usedTcpGlobalPorts | % { $tcpRangesArray += [pscustomobject]@{P1 = $_; P2 = $_ } }
33+
# Sort the list and remove duplicates
34+
$tcpRangesArray = ($tcpRangesArray | Sort-Object { $_.P1 } -Unique)
35+
36+
$tcpRangesList = New-Object System.Collections.ArrayList($null)
37+
$tcpRangesList.AddRange($tcpRangesArray)
38+
39+
# Remove overlapping ranges
40+
for ($i = $tcpRangesList.P1.Length - 2; $i -gt 0 ; $i--) {
41+
if ($tcpRangesList[$i].P2 -gt $tcpRangesList[$i + 1].P1 ) {
42+
$tcpRangesList.Remove($tcpRangesList[$i + 1])
43+
$i++
44+
}
45+
}
46+
47+
# Remove the non-ephemeral port reservations from the list
48+
$filteredTcpRangeArray = $tcpRangesList | ? { $_.P1 -ge $EphemeralPortStart }
49+
$filteredTcpRangeArray = $filteredTcpRangeArray | ? { $_.P2 -le $EphemeralPortEnd }
50+
51+
if ($null -eq $filteredTcpRangeArray) {
52+
$freeRanges = @($EphemeralPortRange[1])
53+
}
54+
else {
55+
$freeRanges = @()
56+
# The first free range goes from $EphemeralPortStart to the beginning of the first reserved range
57+
$freeRanges += ([Convert]::ToUInt32($filteredTcpRangeArray[0].P1) - $EphemeralPortStart)
58+
59+
for ($i = 1; $i -lt $filteredTcpRangeArray.length; $i++) {
60+
# Subsequent free ranges go from the end of the previous reserved range to the beginning of the current reserved range
61+
$freeRanges += ([Convert]::ToUInt32($filteredTcpRangeArray[$i].P1) - [Convert]::ToUInt32($filteredTcpRangeArray[$i - 1].P2) - 1)
62+
}
63+
64+
# The last free range goes from the end of the last reserved range to $EphemeralPortEnd
65+
$freeRanges += ($EphemeralPortEnd - [Convert]::ToUInt32($filteredTcpRangeArray[$filteredTcpRangeArray.length - 1].P2))
66+
}
67+
68+
# Count the number of available free ranges
69+
[uint32]$freeRangesCount = 0
70+
($freeRanges | % { $freeRangesCount += [Math]::Floor($_ / $portRangeSize) } )
71+
72+
return $freeRangesCount
73+
}
74+
75+
function CheckNetworkingServices {
76+
Write-Host "Checking Status of HNS and Kubeproxy"
77+
78+
$statusMessage = ""
79+
$hnsStatus = (Get-Service hns).Status
80+
$kubeProxyStatus = (Get-Service kubeproxy).Status
81+
82+
if($hnsStatus -ne "Running") {
83+
$statusMessage = "HNS is not running. HNS Status : $hnsStatus . Restart hns : Restart-Service -f hns"
84+
}
85+
86+
if($kubeProxyStatus -ne "Running") {
87+
$statusMessage += "KubeProxy is not running. KubeProxy Status : $kubeProxyStatus . Restart KubeProxy : Restart-Service -f kubeproxy"
88+
}
89+
90+
if($statusMessage -eq "") {
91+
Write-Host "HNS and Kubeproxy is running fine" -ForegroundColor Green
92+
return $false
93+
}
94+
95+
Write-Host "$statusMessage" -ForegroundColor Red
96+
97+
return $true
98+
}
99+
100+
function CheckHnsDnsRuleMissing {
101+
$expectedDnsRuleCount = 2
102+
Write-Host "Checking HNS DNS Rule missing"
103+
$dnsRuleCount = ((Get-HnsPolicyList).Policies | where InternalPort -EQ 53 | where ExternalPort -EQ 53).Count
104+
if($dnsRuleCount -lt $expectedDnsRuleCount) {
105+
Write-Host "HNS DNS rule count is $dnsRuleCount. DNS issue for sure." -ForegroundColor Red
106+
Write-Host "Resolution: Upgrade to 1.24.10+, 1.25.6+, 1.26.1+, 1.27.0+" -ForegroundColor Red
107+
Write-Host "Mitigation : Restart-Service -f kubeproxy" -ForegroundColor Red
108+
return $true
109+
}
110+
Write-Host "HNS DNS rule count is $dnsRuleCount. No DNS issue due to missing HNS DNS rules." -ForegroundColor Green
111+
return $false
112+
}
113+
114+
function CheckHnsDeadlock {
115+
Write-Host "Checking HNS Deadlock."
116+
$hnsThreadThrshold = 100
117+
$hnsProcessId = Get-WmiObject -Class Win32_Service -Filter "Name LIKE 'Hns'" | Select-Object -ExpandProperty ProcessId
118+
$hnsThreads = (Get-Process -Id $hnsProcessId).Threads
119+
$threadCount = $hnsThreads.Count
120+
if($threadCount -ge $hnsThreadThrshold) {
121+
Write-Host "HNS thread count is $threadCount which is greater than expected $hnsThreadThrshold. There are chances of deadlock." -ForegroundColor Red
122+
Write-Host "Resolution: Upgrade to Windows 2022" -ForegroundColor Red
123+
Write-Host "Mitigation : Restart-Service -f hns , Start-Sleep -Seconds 10 ; Restart-Service -f KubeProxy " -ForegroundColor Red
124+
return $true
125+
}
126+
Write-Host "HNS thread count is $threadCount . No chances of deadlock." -ForegroundColor Green
127+
return $false
128+
}
129+
130+
function CheckHnsCrash {
131+
Write-Host "Checking HNS crash"
132+
$hnsCrashCount = (Get-WinEvent -FilterHashtable @{logname = 'System'; ProviderName = 'Service Control Manager' } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Where-Object Message -like "*The Host Network Service terminated unexpectedly*").Count
133+
if($hnsCrashCount -gt 0) {
134+
Write-Host "HNS crash count is $hnsCrashCount. There are chances of issues." -ForegroundColor Red
135+
Write-Host "Resolution: Upgrade to 1.24.10+, 1.25.6+, 1.26.1+, 1.27.0+" -ForegroundColor Red
136+
Write-Host "Mitigation : Restart-Service -f KubeProxy " -ForegroundColor Red
137+
return $true
138+
}
139+
Write-Host "HNS crash count is $hnsCrashCount. No issue reported with HNS crash." -ForegroundColor Green
140+
return $false
141+
}
142+
143+
function CheckPortExhaustion {
144+
Write-Host "Checking Port Exhaustion"
145+
$avTcpPorts = CountAvailableEphemeralPorts -protocol TCP
146+
if($avTcpPorts -lt 10) {
147+
Write-Host "Available TCP ports are $avTcpPorts. Port exhaustion suspected." -ForegroundColor Red
148+
return $true
149+
}
150+
$avUdpPorts = CountAvailableEphemeralPorts -protocol UDP
151+
if($avTcpPorts -lt 10) {
152+
Write-Host "Available UDP ports are $avUdpPorts. Port exhaustion suspected." -ForegroundColor Red
153+
return $true
154+
}
155+
Write-Host "Available TCP Ports : $avTcpPorts , UDP Ports : $avUdpPorts . No port exhaustion suspected." -ForegroundColor Green
156+
return $false
157+
}
158+
159+
function CheckKubeProxyCrash {
160+
Write-Host "Checking KubeProxy restart"
161+
for($i = 1; $i -le 10; $i++) {
162+
$status = (Get-Service kubeproxy).Status
163+
if($status -eq "Stopped") {
164+
Write-Host "KubeProxy is restarting. There are chances of issues." -ForegroundColor Red
165+
Write-Host "Resolution: Upgrade to v1.24.12+, v1.25.8, v1.26.3+, v1.27.0+" -ForegroundColor Red
166+
Write-Host "Mitigation : Restart the node or drain to a new node " -ForegroundColor Red
167+
return $true
168+
}
169+
$waitTime = (10 - $i)
170+
Write-Host "Checking KubeProxy restart. Wait time : $waitTime seconds"
171+
Start-Sleep -Seconds 1
172+
}
173+
Write-Host "KubeProxy service state is $status . No issues identified with KubeProxy restart." -ForegroundColor Green
174+
return $false
175+
}
176+
177+
function CheckVfpDnsRuleMissing {
178+
Write-Host "Checking VFP DNS Rule missing"
179+
$vfpDnsRuleMissing = $false
180+
$endpoints = Get-HnsEndpoint
181+
foreach($ep in $endpoints) {
182+
if($ep.IsRemoteEndpoint -eq $true) {
183+
# Write-Host "REP found : $ep"
184+
continue
185+
}
186+
$epID = $ep.ID
187+
$epMac = $ep.MacAddress
188+
$epIpAddress = $ep.IPAddress
189+
$portID = $ep.Resources.Allocators[0].EndpointPortGuid
190+
$tcpRule = vfpctrl.exe /port $portID /layer LB_DSR /group LB_DSR_IPv4_OUT /list-rule | Select-String -Pattern "RULE.*53_53_6"
191+
if($tcpRule.Count -lt 1) {
192+
$vfpDnsRuleMissing = $true
193+
Write-Host "VFP DNS TCP Rule missing for VFP Port : $portID . Endpoint ID : $epID , Mac : $epMac , IP Address : $epIpAddress" -ForegroundColor Red
194+
}
195+
$udpRule = vfpctrl.exe /port $portID /layer LB_DSR /group LB_DSR_IPv4_OUT /list-rule | Select-String -Pattern "RULE.*53_53_17"
196+
if($udpRule.Count -lt 1) {
197+
$vfpDnsRuleMissing = $true
198+
Write-Host "VFP DNS UDP Rule missing for VFP Port : $portID . Endpoint ID : $epID , Mac : $epMac , IP Address : $epIpAddress" -ForegroundColor Red
199+
}
200+
}
201+
202+
if($vfpDnsRuleMissing){
203+
Write-Host "Mitigation : Restart-Service -f hns " -ForegroundColor Red
204+
return $true
205+
}
206+
207+
Write-Host "No issues identified with VFP DNS Rule Missing for local endpoints." -ForegroundColor Green
208+
return $false
209+
}
210+
211+
function DnsPktCapture {
212+
$pktmonLogs = "C:\k\pktmonLogs"
213+
$captureTime = 15
214+
pktmon stop
215+
Write-Host "Starting DNS Packet Capture"
216+
Write-Host "Removing all pktmon filters if anything existing..."
217+
pktmon filter remove
218+
Write-Host "Create DNS Port filter..."
219+
pktmon filter add DNSFilter -p 53
220+
Write-Host "Create a directory for pktmon logs..."
221+
remove-item -Recurse -Force $pktmonLogs -ErrorAction Ignore
222+
mkdir $pktmonLogs
223+
Set-Location $pktmonLogs
224+
Write-Host "Start pktmon. Command : [pktmon start -c --comp all --pkt-size 0 -m multi-file] ..."
225+
pktmon start -c --comp all --pkt-size 0 -m multi-file
226+
Write-Host "Waiting for $captureTime seconds."
227+
Start-Sleep -Seconds $captureTime
228+
pktmon stop
229+
Write-Host "Logs will be available in $pktmonLogs"
230+
Write-Host "DNS Packet Capture Completed"
231+
}
232+
233+
function ValidateNetworkIssues {
234+
Write-Host "Checking Network Issue."
235+
if(CheckNetworkingServices) {
236+
Write-Host "Network Issue Found." -ForegroundColor Red
237+
return $true
238+
}
239+
if(CheckHnsDnsRuleMissing) {
240+
Write-Host "Network Issue Found." -ForegroundColor Red
241+
return $true
242+
}
243+
if(CheckHnsDeadlock) {
244+
Write-Host "Network Issue Found." -ForegroundColor Red
245+
return $true
246+
}
247+
if(CheckHnsCrash) {
248+
Write-Host "Network Issue Found." -ForegroundColor Red
249+
return $true
250+
}
251+
if(CheckPortExhaustion) {
252+
Write-Host "Network Issue Found." -ForegroundColor Red
253+
return $true
254+
}
255+
if(CheckKubeProxyCrash) {
256+
Write-Host "Network Issue Found." -ForegroundColor Red
257+
return $true
258+
}
259+
if(CheckVfpDnsRuleMissing) {
260+
Write-Host "Network Issue Found." -ForegroundColor Red
261+
return $true
262+
}
263+
Write-Host "No Network Issues identified as per current test." -ForegroundColor Green
264+
}
265+
266+
$nwIssueFound = ValidateNetworkIssues
267+
if($nwIssueFound -and $DnsPktCap) {
268+
DnsPktCapture
269+
}

0 commit comments

Comments
 (0)