67 lines
1.5 KiB
Go
67 lines
1.5 KiB
Go
|
package dao
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"io/ioutil"
|
||
|
|
"net/http"
|
||
|
|
"net/url"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/json-iterator/go"
|
||
|
|
|
||
|
|
"go-common/app/job/bbq/recall/internal/conf"
|
||
|
|
"go-common/app/job/bbq/recall/internal/model"
|
||
|
|
)
|
||
|
|
|
||
|
|
func (d *Dao) queryHDFS(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *[]byte, err error) {
|
||
|
|
dt := time.Now().Format("2006-01-02 15:04:05")
|
||
|
|
sign := d.berserkerSign(key.AppKey, key.Secret, dt, "1.0")
|
||
|
|
|
||
|
|
params := &url.Values{}
|
||
|
|
params.Set("appKey", key.AppKey)
|
||
|
|
params.Set("timestamp", dt)
|
||
|
|
params.Set("version", "1.0")
|
||
|
|
params.Set("signMethod", "md5")
|
||
|
|
params.Set("sign", sign)
|
||
|
|
|
||
|
|
fileSuffix := struct {
|
||
|
|
FileSuffix string `json:"fileSuffix"`
|
||
|
|
}{
|
||
|
|
FileSuffix: suffix,
|
||
|
|
}
|
||
|
|
j, err := jsoniter.Marshal(fileSuffix)
|
||
|
|
params.Set("query", string(j))
|
||
|
|
|
||
|
|
for retry := 0; retry < 3; retry++ {
|
||
|
|
resp, err := http.DefaultClient.Get(api + "?" + params.Encode())
|
||
|
|
if err != nil {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
|
||
|
|
b, err := ioutil.ReadAll(resp.Body)
|
||
|
|
resp.Body.Close()
|
||
|
|
if err == nil && len(b) > 0 {
|
||
|
|
result = &b
|
||
|
|
break
|
||
|
|
}
|
||
|
|
// sleep 5s berserker limit
|
||
|
|
time.Sleep(5 * time.Second)
|
||
|
|
}
|
||
|
|
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (d *Dao) scanHDFSPath(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *model.HDFSResult, err error) {
|
||
|
|
b, err := d.queryHDFS(c, api, key, suffix)
|
||
|
|
if err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
result = &model.HDFSResult{}
|
||
|
|
err = jsoniter.Unmarshal(*b, result)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (d *Dao) loadHDFSFile(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *[]byte, err error) {
|
||
|
|
return d.queryHDFS(c, api, key, suffix)
|
||
|
|
}
|