better-staridc-MNBT
1# Parallel resumable fetcher for http://w2.test.idcsmart.com/doc/info
2# Fetches the missing raw/*.html files using N parallel Start-Job workers.
3# Usage: powershell -ExecutionPolicy Bypass -File fetch_parallel.ps1
4# Then re-run fetch_api_docs.ps1 (it skips existing files and generates zjmf-api.md).
5#
6# NOTE: ASCII-only on purpose (avoids PS5.1 script-encoding quirks).
7
8param(
9 [string]$BaseUrl = 'http://w2.test.idcsmart.com',
10 [int]$Jobs = 8
11)
12
13$ErrorActionPreference = 'Continue'
14$OutDir = Split-Path -Parent $MyInvocation.MyCommand.Path
15$RawDir = Join-Path $OutDir 'raw'
16$ListPath = Join-Path $OutDir 'doc_list.json'
17
18if (-not (Test-Path $ListPath)) {
19 Write-Host "missing $ListPath ; run fetch_api_docs.ps1 first"
20 exit 1
21}
22
23# ---------- flatten all action names ----------
24$tree = Get-Content $ListPath -Raw -Encoding UTF8 | ConvertFrom-Json
25$actions = New-Object System.Collections.ArrayList
26function Add-Actions($nodes) {
27 foreach ($n in $nodes) {
28 if ($n.actions) {
29 foreach ($a in $n.actions) {
30 if ($a.name) { [void]$actions.Add([string]$a.name) }
31 }
32 }
33 if ($n.children) { Add-Actions $n.children }
34 }
35}
36Add-Actions $tree.list
37
38function Get-RawPath([string]$name) {
39 return Join-Path $RawDir ((( $name ) -replace '[^A-Za-z0-9._-]', '_') + '.html')
40}
41
42$missing = @($actions | Where-Object { -not (Test-Path (Get-RawPath $_)) })
43
44Write-Host ("actions: {0}, already fetched: {1}, missing: {2}" -f $actions.Count, ($actions.Count - $missing.Count), $missing.Count)
45if ($missing.Count -eq 0) {
46 Write-Host 'nothing to fetch'
47 exit 0
48}
49
50# ---------- split into chunks ----------
51$size = [Math]::Ceiling($missing.Count / $Jobs)
52$chunks = New-Object System.Collections.ArrayList
53for ($i = 0; $i -lt $missing.Count; $i += $size) {
54 $end = [Math]::Min($i + $size - 1, $missing.Count - 1)
55 $slice = @($missing[$i..$end])
56 [void]$chunks.Add($slice)
57}
58Write-Host ("workers: {0}, chunk size: {1}" -f $chunks.Count, $size)
59
60# ---------- worker ----------
61$worker = {
62 param($names, $base, $rawDir)
63 $ErrorActionPreference = 'Continue'
64 $done = 0
65 foreach ($name in $names) {
66 $f = Join-Path $rawDir ((($name) -replace '[^A-Za-z0-9._-]', '_') + '.html')
67 if (Test-Path $f) { continue }
68 $ok = $false
69 for ($t = 0; $t -lt 3 -and -not $ok; $t++) {
70 try {
71 $r = Invoke-WebRequest -Uri ($base + '/doc/info?name=' + $name) -UseBasicParsing -TimeoutSec 30
72 [System.IO.File]::WriteAllText($f, $r.Content, [System.Text.Encoding]::UTF8)
73 $ok = $true
74 } catch {
75 Start-Sleep -Milliseconds 400
76 }
77 }
78 $done++
79 if (($done % 100) -eq 0) { Write-Host (" worker progress {0}" -f $done) }
80 Start-Sleep -Milliseconds 10
81 }
82 Write-Host ("worker finished {0}" -f $names.Count)
83}
84
85foreach ($c in $chunks) {
86 Start-Job -ScriptBlock $worker -ArgumentList $c, $BaseUrl, $RawDir | Out-Null
87}
88
89Get-Job | Wait-Job | Out-Null
90
91foreach ($j in Get-Job) {
92 Receive-Job $j
93 Remove-Job $j -Force
94}
95
96# ---------- verify ----------
97$stillMissing = @($actions | Where-Object { -not (Test-Path (Get-RawPath $_)) })
98Write-Host ("verify: missing {0}/{1}" -f $stillMissing.Count, $actions.Count)