better-staridc-MNBT
1# zjmf-api-doc crawler for http://w2.test.idcsmart.com/doc
2# Usage: powershell -ExecutionPolicy Bypass -File fetch_api_docs.ps1
3# Outputs (same dir):
4# doc_list.json raw api catalog (JSON)
5# raw/*.html per-action detail pages (resumable; existing files are skipped)
6# zjmf-api.md merged Markdown doc
7#
8# NOTE: this script is ASCII-only on purpose (avoids PS5.1 script-encoding quirks).
9# The doc content itself is Chinese because it comes from the crawled pages.
10
11param(
12 [string]$BaseUrl = 'http://w2.test.idcsmart.com',
13 [int]$SleepMs = 80,
14 [int]$MaxRetries = 3
15)
16
17$ErrorActionPreference = 'Continue'
18$OutDir = Split-Path -Parent $MyInvocation.MyCommand.Path
19$RawDir = Join-Path $OutDir 'raw'
20$ListPath = Join-Path $OutDir 'doc_list.json'
21$MdPath = Join-Path $OutDir 'zjmf-api.md'
22New-Item -ItemType Directory -Force -Path $RawDir | Out-Null
23
24function Invoke-WithRetry([string]$url) {
25 for ($i = 0; $i -lt $MaxRetries; $i++) {
26 try {
27 return Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec 30
28 } catch {
29 Start-Sleep -Milliseconds 800
30 }
31 }
32 Write-Warning "fetch failed: $url"
33 return $null
34}
35
36# ---------- 1. catalog ----------
37if (-not (Test-Path $ListPath)) {
38 Write-Host 'Fetching /doc/list ...'
39 $r = Invoke-WithRetry "$BaseUrl/doc/list"
40 if ($r) {
41 [System.IO.File]::WriteAllText($ListPath, $r.Content, [System.Text.Encoding]::UTF8)
42 }
43}
44$tree = Get-Content $ListPath -Raw -Encoding UTF8 | ConvertFrom-Json
45
46# ---------- 2. flatten all actions (multi-level children) ----------
47$actions = New-Object System.Collections.ArrayList
48function Add-Actions($nodes) {
49 foreach ($n in $nodes) {
50 if ($n.actions) {
51 foreach ($a in $n.actions) {
52 if ($a.name) {
53 [void]$actions.Add([pscustomobject]@{
54 name = $a.name
55 title = $a.title
56 url = $a.url
57 method = $a.method
58 desc = $a.desc
59 params = $a.param
60 rets = $a.return
61 })
62 }
63 }
64 }
65 if ($n.children) { Add-Actions $n.children }
66 }
67}
68Add-Actions $tree.list
69$total = $actions.Count
70Write-Host "Total actions: $total"
71
72# ---------- 3. fetch detail pages (resumable) ----------
73function Get-SafeName([string]$name) {
74 return ($name -replace '[^A-Za-z0-9._-]', '_')
75}
76
77$done = 0
78foreach ($a in $actions) {
79 $rawFile = Join-Path $RawDir ((Get-SafeName $a.name) + '.html')
80 if (Test-Path $rawFile) { $done++; continue }
81 $r = Invoke-WithRetry ("$BaseUrl/doc/info?name=" + $a.name)
82 if ($r) {
83 [System.IO.File]::WriteAllText($rawFile, $r.Content, [System.Text.Encoding]::UTF8)
84 }
85 $done++
86 if (($done % 25) -eq 0) { Write-Host ("progress {0}/{1}" -f $done, $total) }
87 Start-Sleep -Milliseconds $SleepMs
88}
89Write-Host "Fetch done: $done/$total"
90
91# ---------- 4. parse one detail page ----------
92function Parse-DocHtml([string]$html) {
93 $doc = [pscustomobject]@{ title = ''; url = ''; method = ''; desc = ''; json = ''; params = @() }
94 if (-not $html) { return $doc }
95 if ($html -match '<h2>[^<]*([^<]*)</h2>') { $doc.title = $Matches[1].Trim() }
96 if ($html -match 'class="text-primary">([^<]*)</p>') { $doc.desc = $Matches[1].Trim() }
97 if ($html -match '<code id="json_text">(.*?)</code>') {
98 $doc.json = ([System.Net.WebUtility]::HtmlDecode($Matches[1]) -replace '<br\s*/?>', "`n").Trim()
99 }
100 # url + method: line like: <p>[address] <span class="label label-success">POST</span></p>
101 $pMatch = [regex]::Match($html, '<p>[^<]*?\s+([^<]+?)\s*<span class="label[^"]*"[^>]*>\s*([^<]*?)\s*</span></p>', 'Singleline')
102 if ($pMatch.Success) {
103 $doc.url = $pMatch.Groups[1].Value.Trim()
104 $doc.method = $pMatch.Groups[2].Value.Trim()
105 }
106 # param table after <h3>
107 $tblMatch = [regex]::Match($html, '<h3>[^<]*</h3>\s*<table[^>]*>(.*?)</table>', 'Singleline')
108 if ($tblMatch.Success) {
109 foreach ($row in [regex]::Matches($tblMatch.Groups[1].Value, '<tr>(.*?)</tr>', 'Singleline')) {
110 $cells = [regex]::Matches($row.Groups[1].Value, '<td>(.*?)</td>', 'Singleline')
111 if ($cells.Count -ge 2) {
112 $vals = @()
113 foreach ($c in $cells) {
114 $vals += [System.Net.WebUtility]::HtmlDecode($c.Groups[1].Value).Trim()
115 }
116 $doc.params += , $vals
117 }
118 }
119 }
120 return $doc
121}
122
123function Get-ActionDetail($action) {
124 $rawFile = Join-Path $RawDir ((Get-SafeName $action.name) + '.html')
125 if (Test-Path $rawFile) {
126 return Parse-DocHtml (Get-Content $rawFile -Raw -Encoding UTF8)
127 }
128 return $null
129}
130
131# ---------- 5. generate Markdown ----------
132$sb = New-Object System.Text.StringBuilder
133[void]$sb.AppendLine('# zjmf finance API doc')
134[void]$sb.AppendLine('')
135[void]$sb.AppendLine(('> crawled from ' + $BaseUrl + '/doc at ' + (Get-Date -Format 'yyyy-MM-dd HH:mm') + ' | actions: ' + $total))
136[void]$sb.AppendLine('> sources: doc_list.json + raw/*.html | re-crawl: powershell -ExecutionPolicy Bypass -File fetch_api_docs.ps1')
137[void]$sb.AppendLine('')
138
139function Write-ActionMd($a) {
140 $d = Get-ActionDetail $a
141 $title = ''
142 $method = ''
143 $url = ''
144 $desc = ''
145 if ($d) {
146 $title = $d.title
147 $method = $d.method
148 $url = $d.url
149 $desc = $d.desc
150 }
151 if (-not $title) { $title = $a.title }
152 if (-not $method) { $method = $a.method }
153 if (-not $url) { $url = $a.url }
154 if (-not $desc) { $desc = $a.desc }
155
156 $heading = '### ' + $title
157 if ($method) { $heading += ' -- ' + $method.ToUpper() }
158 if ($url) { $heading += ' ' + $url }
159 [void]$sb.AppendLine($heading)
160 [void]$sb.AppendLine('')
161 [void]$sb.AppendLine('- controller: ``' + $a.name + '``')
162 if ($desc) { [void]$sb.AppendLine('- desc: ' + $desc) }
163 [void]$sb.AppendLine('')
164
165 $paramRows = $null
166 if ($d -and $d.params -and $d.params.Count -gt 0) { $paramRows = $d.params }
167 elseif ($a.params -and $a.params.Count -gt 0) { $paramRows = $a.params }
168
169 if ($paramRows -and $paramRows.Count -gt 0) {
170 [void]$sb.AppendLine('**params**')
171 [void]$sb.AppendLine('')
172 [void]$sb.AppendLine('| name | type | required | default | other | desc |')
173 [void]$sb.AppendLine('| --- | --- | --- | --- | --- | --- |')
174 $isList = $paramRows[0] -is [System.Array]
175 if ($isList) {
176 foreach ($p in $paramRows) {
177 $cells = @()
178 foreach ($v in $p) { $cells += ('' + $v) -replace "`n", ' ' }
179 $line = '| ' + ($cells -join ' | ') + ' |'
180 [void]$sb.AppendLine($line)
181 }
182 } else {
183 foreach ($p in $paramRows) {
184 $f = @($p.name, $p.type, $p.require, $p.default, $p.other, $p.desc)
185 [void]$sb.AppendLine('| ' + ($f -join ' | ') + ' |')
186 }
187 }
188 [void]$sb.AppendLine('')
189 }
190
191 if ($d -and $d.json) {
192 [void]$sb.AppendLine('**response example**')
193 [void]$sb.AppendLine('')
194 [void]$sb.AppendLine('```json')
195 [void]$sb.AppendLine($d.json)
196 [void]$sb.AppendLine('```')
197 [void]$sb.AppendLine('')
198 }
199
200 if ($a.rets) {
201 foreach ($r in @($a.rets)) {
202 if ($r) { [void]$sb.AppendLine('- return-field: ' + $r) }
203 }
204 [void]$sb.AppendLine('')
205 }
206}
207
208function Walk-ForMd($nodes) {
209 foreach ($n in $nodes) {
210 if ($n.actions -and $n.actions.Count -gt 0) {
211 [void]$sb.AppendLine('---')
212 [void]$sb.AppendLine('')
213 [void]$sb.AppendLine('## ' + $n.title)
214 [void]$sb.AppendLine('')
215 foreach ($a in $n.actions) {
216 if ($a.name) { Write-ActionMd $a }
217 }
218 [void]$sb.AppendLine('')
219 }
220 if ($n.children) { Walk-ForMd $n.children }
221 }
222}
223Walk-ForMd $tree.list
224
225[System.IO.File]::WriteAllText($MdPath, $sb.ToString(), [System.Text.Encoding]::UTF8)
226Write-Host "Generated $MdPath"
227Write-Host 'All done'