Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 3ea965b

Browse filesBrowse files
colinz2colinz2
committed
init, test leetcode-cn.com api
1 parent 4d0cbe2 commit 3ea965b
Copy full SHA for 3ea965b

File tree

Expand file treeCollapse file tree

4 files changed

+308
-0
lines changed
Filter options
Expand file treeCollapse file tree

4 files changed

+308
-0
lines changed

‎aid/client.go

Copy file name to clipboard
+184Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
package aid
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"errors"
7+
"fmt"
8+
"io"
9+
"net"
10+
"net/http"
11+
"net/url"
12+
"sync"
13+
"time"
14+
15+
"github.com/machinebox/graphql"
16+
)
17+
18+
const (
19+
Url = "https://leetcode-cn.com"
20+
LoginPath = "/accounts/login"
21+
SubmissionsPath = "/api/submissions"
22+
GraphqlPath = "/graphql"
23+
24+
UAStr = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " +
25+
"Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134"
26+
)
27+
28+
func setHeader(header *http.Header) {
29+
header.Set("User-Agent", UAStr)
30+
header.Set("Origin", Url)
31+
}
32+
33+
type Client struct {
34+
conf ClientConf
35+
httpCli *http.Client
36+
cookieJar *http.CookieJar
37+
}
38+
39+
type ClientConf struct {
40+
UserName string
41+
PassWord string
42+
}
43+
44+
// Jar https://stackoverflow.com/questions/12756782/go-http-post-and-use-cookies
45+
type Jar struct {
46+
lk sync.Mutex
47+
cookies map[string][]*http.Cookie
48+
}
49+
50+
func NewJar() *Jar {
51+
jar := new(Jar)
52+
jar.cookies = make(map[string][]*http.Cookie)
53+
return jar
54+
}
55+
56+
func (jar *Jar) SetCookies(u *url.URL, cookies []*http.Cookie) {
57+
jar.lk.Lock()
58+
jar.cookies[u.Host] = cookies
59+
jar.lk.Unlock()
60+
}
61+
62+
func (jar *Jar) Cookies(u *url.URL) []*http.Cookie {
63+
return jar.cookies[u.Host]
64+
}
65+
66+
func NewClient(conf *ClientConf) *Client {
67+
httpClient := &http.Client{
68+
Transport: &http.Transport{
69+
DialContext: (&net.Dialer{
70+
Timeout: 10 * time.Second,
71+
KeepAlive: 10 * time.Second,
72+
}).DialContext,
73+
TLSHandshakeTimeout: 10 * time.Second,
74+
75+
ExpectContinueTimeout: 4 * time.Second,
76+
ResponseHeaderTimeout: 3 * time.Second,
77+
},
78+
// Prevent endless redirects
79+
Timeout: 1 * time.Minute,
80+
Jar: NewJar(),
81+
}
82+
83+
return &Client{
84+
conf: *conf,
85+
httpCli: httpClient,
86+
}
87+
}
88+
89+
type logInParam struct {
90+
Login string `json:"login"`
91+
Password string `json:"password"`
92+
}
93+
94+
func (l logInParam) postFormBuffer() *bytes.Buffer {
95+
postFormValues := url.Values{}
96+
postFormValues.Add("login", l.Login)
97+
postFormValues.Add("password", l.Password)
98+
return bytes.NewBufferString(postFormValues.Encode())
99+
}
100+
101+
func (c *Client) Login(ctx context.Context) error {
102+
loginUrl := Url + LoginPath
103+
logInParam := logInParam{
104+
Login: c.conf.UserName,
105+
Password: c.conf.PassWord,
106+
}
107+
108+
postReq, err := http.NewRequest(http.MethodPost, loginUrl, logInParam.postFormBuffer())
109+
if err != nil {
110+
return err
111+
}
112+
setHeader(&postReq.Header)
113+
postReq.Header.Set("referer", loginUrl)
114+
postReq.Header.Set("x-requested-with", "XMLHttpRequest")
115+
// content-type must be right: application/x-www-form-urlencoded
116+
postReq.Header.Set("content-type", "application/x-www-form-urlencoded")
117+
118+
response, err := c.httpCli.Do(postReq)
119+
if err != nil {
120+
return err
121+
}
122+
defer response.Body.Close()
123+
124+
if response.StatusCode != http.StatusOK {
125+
return errors.New(fmt.Sprintf("http status not OK, %d", response.StatusCode))
126+
}
127+
return nil
128+
}
129+
130+
func (c *Client) GetSubmission(ctx context.Context) error {
131+
submissionUrl := Url + SubmissionsPath + "?offset=0&limit=2000"
132+
getReq, err := http.NewRequestWithContext(ctx, http.MethodGet, submissionUrl, nil)
133+
if err != nil {
134+
return err
135+
}
136+
setHeader(&getReq.Header)
137+
getReq.Header.Set("content-type", "application/json")
138+
139+
getResponse, err := c.httpCli.Do(getReq)
140+
if err != nil {
141+
return err
142+
}
143+
defer getResponse.Body.Close()
144+
145+
responseBody, err := io.ReadAll(getResponse.Body)
146+
if err != nil {
147+
return err
148+
}
149+
fmt.Println(string(responseBody))
150+
151+
return nil
152+
}
153+
154+
func (c *Client) GraphqlTest() {
155+
graphqlUrl := Url + GraphqlPath
156+
graphqlCli := graphql.NewClient(graphqlUrl, graphql.WithHTTPClient(c.httpCli))
157+
158+
req := graphql.NewRequest(`
159+
query questionData($titleSlug: String!) {
160+
question(titleSlug: $titleSlug) {
161+
questionId
162+
content
163+
translatedTitle
164+
translatedContent
165+
similarQuestions
166+
topicTags {
167+
name
168+
slug
169+
translatedName
170+
}
171+
hints
172+
}
173+
}
174+
`)
175+
req.Var("titleSlug", "two-sum")
176+
var responseData map[string]interface{}
177+
setHeader(&req.Header)
178+
req.Header.Set("Cache-Control", "no-cache")
179+
err := graphqlCli.Run(context.TODO(), req, &responseData)
180+
if err != nil {
181+
panic(err)
182+
}
183+
fmt.Println(responseData)
184+
}

‎aid/scraping.go

Copy file name to clipboard
+45Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package aid
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"fmt"
7+
"github.com/gocolly/colly"
8+
"log"
9+
"net/http"
10+
)
11+
12+
func Scraping(conf *ClientConf) {
13+
c := colly.NewCollector()
14+
15+
c.OnError(func(response *colly.Response, err error) {
16+
fmt.Println("on error:", err, response.StatusCode)
17+
fmt.Println(string(response.Body))
18+
})
19+
20+
logInParam := logInParam{
21+
Login: conf.UserName,
22+
Password: conf.PassWord,
23+
}
24+
reqBody, err := json.Marshal(&logInParam)
25+
if err != nil {
26+
panic(err)
27+
}
28+
29+
// authenticate
30+
loginUrl := Url + LoginPath
31+
header := http.Header{}
32+
header.Set("Referer", loginUrl)
33+
err = c.Request(http.MethodPost, loginUrl, bytes.NewReader(reqBody), nil, header)
34+
if err != nil {
35+
log.Fatal("error:", err)
36+
}
37+
38+
// attach callbacks after login
39+
c.OnResponse(func(r *colly.Response) {
40+
log.Println("response received", r.StatusCode)
41+
})
42+
43+
// start scraping
44+
c.Visit(Url + SubmissionsPath)
45+
}

‎go.mod

Copy file name to clipboard
+19Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
module github.com/realzhangm/leetcode_aid
2+
3+
go 1.16
4+
5+
require (
6+
github.com/PuerkitoBio/goquery v1.7.1 // indirect
7+
github.com/antchfx/htmlquery v1.2.4 // indirect
8+
github.com/antchfx/xmlquery v1.3.8 // indirect
9+
github.com/gobwas/glob v0.2.3 // indirect
10+
github.com/gocolly/colly v1.2.0
11+
github.com/kennygrant/sanitize v1.2.4 // indirect
12+
github.com/machinebox/graphql v0.2.2
13+
github.com/matryer/is v1.4.0 // indirect
14+
github.com/pkg/errors v0.9.1 // indirect
15+
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
16+
github.com/temoto/robotstxt v1.1.2 // indirect
17+
golang.org/x/net v0.0.0-20211020060615-d418f374d309 // indirect
18+
google.golang.org/appengine v1.6.7 // indirect
19+
)

‎go.sum

Copy file name to clipboard
+60Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4=
2+
github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY=
3+
github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
4+
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
5+
github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
6+
github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
7+
github.com/antchfx/xmlquery v1.3.8 h1:dRnBQM3Vk5BVJFvFwsHOLAox+mEiNw5ZusaUNCrEdoU=
8+
github.com/antchfx/xmlquery v1.3.8/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks=
9+
github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
10+
github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
11+
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
12+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
13+
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
14+
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
15+
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
16+
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
17+
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
18+
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
19+
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
20+
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
21+
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
22+
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
23+
github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo=
24+
github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA=
25+
github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE=
26+
github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
27+
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
28+
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
29+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
30+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
31+
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
32+
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
33+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
34+
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
35+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
36+
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
37+
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
38+
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
39+
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
40+
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
41+
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
42+
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
43+
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
44+
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
45+
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
46+
golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
47+
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
48+
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
49+
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
50+
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
51+
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
52+
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
53+
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
54+
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
55+
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
56+
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
57+
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
58+
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
59+
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
60+
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.