294 lines
6.5 KiB
Go
294 lines
6.5 KiB
Go
|
package report
|
||
|
|
||
|
import (
|
||
|
"bytes"
|
||
|
"context"
|
||
|
"encoding/json"
|
||
|
"io"
|
||
|
xhttp "net/http"
|
||
|
"strings"
|
||
|
"sync"
|
||
|
"time"
|
||
|
|
||
|
"go-common/app/job/main/tv/conf"
|
||
|
"go-common/app/job/main/tv/dao/report"
|
||
|
mdlrep "go-common/app/job/main/tv/model/report"
|
||
|
"go-common/library/log"
|
||
|
"go-common/library/sync/pipeline/fanout"
|
||
|
|
||
|
"github.com/robfig/cron"
|
||
|
)
|
||
|
|
||
|
const (
|
||
|
_retry = 3
|
||
|
_jobRunning = 3
|
||
|
_startJob = 4
|
||
|
_readSize = 1024
|
||
|
)
|
||
|
|
||
|
// Service struct of service .
|
||
|
type Service struct {
|
||
|
c *conf.Config
|
||
|
ch chan bool
|
||
|
dao *report.Dao
|
||
|
respURL map[string]interface{}
|
||
|
cache *fanout.Fanout
|
||
|
lock sync.Mutex
|
||
|
labelRes map[int]map[string]int
|
||
|
readSize int
|
||
|
// cron
|
||
|
cron *cron.Cron
|
||
|
}
|
||
|
|
||
|
// New creates a Service instance.
|
||
|
func New(c *conf.Config) (s *Service) {
|
||
|
s = &Service{
|
||
|
c: c,
|
||
|
dao: report.New(c),
|
||
|
respURL: make(map[string]interface{}),
|
||
|
cache: fanout.New("cache", fanout.Worker(1), fanout.Buffer(1024)),
|
||
|
labelRes: make(map[int]map[string]int),
|
||
|
readSize: c.Report.ReadSize * _readSize,
|
||
|
cron: cron.New(),
|
||
|
ch: make(chan bool, c.Report.RoutineCount),
|
||
|
}
|
||
|
if err := s.cron.AddFunc(s.c.Report.CronAc, s.oneWork(mdlrep.ArchiveClick)); err != nil { // corn report run
|
||
|
panic(err)
|
||
|
}
|
||
|
if err := s.cron.AddFunc(s.c.Report.CronAd, s.oneWork(mdlrep.ActiveDuration)); err != nil { // corn report run
|
||
|
panic(err)
|
||
|
}
|
||
|
if err := s.cron.AddFunc(s.c.Report.CronPd, s.oneWork(mdlrep.PlayDuration)); err != nil { // corn report run
|
||
|
panic(err)
|
||
|
}
|
||
|
if err := s.cron.AddFunc(s.c.Report.CronVe, s.oneWork(mdlrep.VisitEvent)); err != nil { // corn report run
|
||
|
panic(err)
|
||
|
}
|
||
|
s.cron.Start()
|
||
|
s.readCache() // data report
|
||
|
s.readLabelCache() // label style
|
||
|
go s.reportCon() // data report
|
||
|
go s.showStyle() // label style
|
||
|
go s.showLabel() // label style
|
||
|
return
|
||
|
}
|
||
|
|
||
|
func (s *Service) oneWork(table string) func() {
|
||
|
return func() {
|
||
|
if s.c.Report.Env != "prod" {
|
||
|
return
|
||
|
}
|
||
|
var (
|
||
|
res string
|
||
|
err error
|
||
|
)
|
||
|
if res, err = s.requestURL(table); err != nil {
|
||
|
log.Error("reportPro s.requestURL() error(%v)", err)
|
||
|
return
|
||
|
}
|
||
|
if res == "" {
|
||
|
return
|
||
|
}
|
||
|
s.lock.Lock()
|
||
|
s.respURL[res] = struct{}{}
|
||
|
s.lock.Unlock()
|
||
|
s.setCache()
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func (s *Service) reportCon() {
|
||
|
if s.c.Report.Env != "prod" {
|
||
|
return
|
||
|
}
|
||
|
var (
|
||
|
err error
|
||
|
info *mdlrep.DpCheckJobResult
|
||
|
)
|
||
|
for {
|
||
|
var (
|
||
|
flags, failStr []string
|
||
|
)
|
||
|
s.lock.Lock()
|
||
|
for k := range s.respURL {
|
||
|
flags = append(flags, k)
|
||
|
}
|
||
|
s.respURL = make(map[string]interface{})
|
||
|
s.lock.Unlock()
|
||
|
for _, v := range flags {
|
||
|
if v == "" {
|
||
|
continue
|
||
|
}
|
||
|
// loop send http request and return result
|
||
|
if info, err = s.check(v); err == nil && len(info.Files) > 0 {
|
||
|
now := time.Now()
|
||
|
s.upReport(info)
|
||
|
log.Warn("report success fileNum(%d) url(%s) 本次上报数据耗时: %s", len(info.Files), v, time.Since(now))
|
||
|
continue
|
||
|
}
|
||
|
if info.StatusID == _jobRunning || info.StatusID == _startJob {
|
||
|
failStr = append(failStr, v)
|
||
|
}
|
||
|
}
|
||
|
s.lock.Lock()
|
||
|
for _, v := range failStr {
|
||
|
s.respURL[v] = struct{}{}
|
||
|
}
|
||
|
s.lock.Unlock()
|
||
|
s.setCache()
|
||
|
time.Sleep(3 * time.Second)
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func (s *Service) readFile(path string) {
|
||
|
var (
|
||
|
n int
|
||
|
err error
|
||
|
resdata []map[string]interface{}
|
||
|
resp *xhttp.Response
|
||
|
buf = make([]byte, 1024)
|
||
|
chunks []byte
|
||
|
req *xhttp.Request
|
||
|
fileCnt = 0
|
||
|
)
|
||
|
client := &xhttp.Client{
|
||
|
Transport: &xhttp.Transport{
|
||
|
DisableKeepAlives: true,
|
||
|
},
|
||
|
}
|
||
|
req, err = xhttp.NewRequest("GET", path, strings.NewReader(""))
|
||
|
if err != nil {
|
||
|
log.Error("[url(%s)] xhttp.NewRequest error(%v)", path, err)
|
||
|
return
|
||
|
}
|
||
|
resp, err = client.Do(req)
|
||
|
if err != nil {
|
||
|
log.Error("[url(%s)] client.Do error(%v)", path, err)
|
||
|
return
|
||
|
}
|
||
|
defer resp.Body.Close()
|
||
|
for {
|
||
|
n, err = resp.Body.Read(buf)
|
||
|
if err != nil {
|
||
|
if err == io.EOF {
|
||
|
break
|
||
|
}
|
||
|
log.Error("resp.Body.Read error(%v)", err)
|
||
|
return
|
||
|
}
|
||
|
if 0 == n {
|
||
|
break
|
||
|
}
|
||
|
chunks = append(chunks, buf[:n]...)
|
||
|
if len(chunks) > s.readSize { // 500K
|
||
|
lastPos := bytes.LastIndex(chunks, []byte("\n"))
|
||
|
if lastPos < 0 {
|
||
|
continue
|
||
|
}
|
||
|
fileCnt = fileCnt + 1
|
||
|
results := append([]byte{}, chunks[:lastPos]...)
|
||
|
chunks = append([]byte{}, chunks[lastPos:]...)
|
||
|
bsdata := bytes.Split(results, []byte("\n"))
|
||
|
for _, bs := range bsdata {
|
||
|
n := bytes.Split(bs, []byte("\u0001"))
|
||
|
m := mdlrep.ArcClickParam(n)
|
||
|
resdata = append(resdata, m)
|
||
|
}
|
||
|
if err = s.postData(resdata); err != nil {
|
||
|
log.Error("[url(%s)] s.postData error(%v)", path, err)
|
||
|
}
|
||
|
resdata = make([]map[string]interface{}, 0)
|
||
|
}
|
||
|
}
|
||
|
if len(chunks) > 0 {
|
||
|
bsdata := bytes.Split(chunks, []byte("\n"))
|
||
|
for _, bs := range bsdata {
|
||
|
n := bytes.Split(bs, []byte("\u0001"))
|
||
|
m := mdlrep.ArcClickParam(n)
|
||
|
resdata = append(resdata, m)
|
||
|
}
|
||
|
if err = s.postData(resdata); err != nil {
|
||
|
log.Error("[url(%s)] s.postData error(%v)", path, err)
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func (s *Service) requestURL(table string) (res string, err error) {
|
||
|
for i := 0; i < _retry; i++ {
|
||
|
if res, err = s.dao.Report(context.Background(), table); err == nil {
|
||
|
break
|
||
|
}
|
||
|
}
|
||
|
return
|
||
|
}
|
||
|
|
||
|
func (s *Service) check(res string) (info *mdlrep.DpCheckJobResult, err error) {
|
||
|
for i := 0; i < _retry; i++ {
|
||
|
if info, err = s.dao.CheckJob(context.Background(), res); err == nil {
|
||
|
break
|
||
|
}
|
||
|
}
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// upReport .
|
||
|
func (s *Service) upReport(info *mdlrep.DpCheckJobResult) {
|
||
|
for _, v := range info.Files {
|
||
|
s.readFile(v)
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func (s *Service) postData(param []map[string]interface{}) (err error) {
|
||
|
for _, v := range param {
|
||
|
s.ch <- true
|
||
|
go s.sendOnce(v)
|
||
|
}
|
||
|
return
|
||
|
}
|
||
|
|
||
|
func (s *Service) sendOnce(v map[string]interface{}) (err error) {
|
||
|
var (
|
||
|
body string
|
||
|
data []byte
|
||
|
)
|
||
|
defer func() {
|
||
|
<-s.ch
|
||
|
}()
|
||
|
if data, err = json.Marshal(v); err != nil {
|
||
|
log.Error("Service postData json.Marshal error(%v)", err)
|
||
|
return
|
||
|
}
|
||
|
body = body + string(data) + ","
|
||
|
s.dealBody(body)
|
||
|
return
|
||
|
}
|
||
|
|
||
|
func (s *Service) readCache() {
|
||
|
if s.c.Report.Env != "prod" {
|
||
|
return
|
||
|
}
|
||
|
var (
|
||
|
err error
|
||
|
btRes = make(map[string]interface{})
|
||
|
)
|
||
|
if btRes, err = s.dao.GetReportCache(context.Background()); err != nil {
|
||
|
log.Error("s.dao.GetReportCache error(%v)", err)
|
||
|
panic(err)
|
||
|
}
|
||
|
s.respURL = btRes
|
||
|
}
|
||
|
|
||
|
func (s *Service) dealBody(body string) {
|
||
|
body = strings.Replace(body, `\\N`, "", -1)
|
||
|
body = strings.TrimSuffix(body, ",")
|
||
|
body = `{"code": 0,"message": "0","ttl": 1,"data":[` + body + `]}`
|
||
|
if err := s.dao.PostRequest(context.Background(), body); err != nil {
|
||
|
log.Error("s.dao.PostRequest error(%v)", err)
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func (s *Service) setCache() {
|
||
|
s.cache.Do(context.Background(), func(c context.Context) {
|
||
|
s.dao.SetReportCache(c, s.respURL)
|
||
|
})
|
||
|
}
|