Create & Init Project...

app/admin/main/push/service/BUILD

@@ -0,0 +1,58 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/push/conf:go_default_library",
+        "//app/admin/main/push/model:go_default_library",
+        "//app/service/main/push/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dataplatform.go",
+        "service.go",
+        "task.go",
+        "upload.go",
+    ],
+    importpath = "go-common/app/admin/main/push/service",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/push/conf:go_default_library",
+        "//app/admin/main/push/dao:go_default_library",
+        "//app/admin/main/push/model:go_default_library",
+        "//app/service/main/push/model:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/jinzhu/gorm:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/push/service/dataplatform.go

@@ -0,0 +1,444 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/admin/main/push/model"
+	pushmdl "go-common/app/service/main/push/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+)
+
+const (
+	_countryForeign = 1
+	_countryChina   = 2
+
+	_sexMale     = 1
+	_sexFemale   = 2
+	_platformAll = "4" // 全部平台,iOS、小米、华为...
+
+	_attentionTypeUnion = 1 // 多个自选关注条件用union查询 (另一种是join查询)
+
+	_timeLayout = "2006-01-02 15:04:05"
+
+	_retry = 3
+
+	// 检查数据平台当天的计算任务有没有成功
+	checkDpDataURL = "http://berserker.bilibili.co/api/archer/project/%d/status"
+)
+
+var (
+	areas = map[int]string{1: "海外", 2: "中国", 3: "青海", 4: "上海", 5: "安徽", 6: "广西", 7: "贵州", 8: "吉林", 9: "福建", 10: "黑龙江",
+		11: "江西", 12: "甘肃", 13: "云南", 14: "湖南", 15: "河北", 16: "山东", 17: "湖北", 18: "广东", 19: "宁夏", 20: "重庆",
+		21: "辽宁", 22: "内蒙古", 23: "山西", 24: "澳门", 25: "陕西", 26: "江苏", 27: "四川", 28: "浙江", 29: "海南", 30: "河南",
+		31: "北京", 32: "香港", 33: "台湾", 34: "天津", 35: "西藏", 36: "新疆", 37: "其他",
+	}
+	dpProjects = map[int]string{
+		355170: "mid维度数据",
+		357214: "buvid维度数据",
+		355628: "自选关注数据",
+	}
+)
+
+// DpTaskInfo .
+func (s *Service) DpTaskInfo(ctx context.Context, id int64, job string) (res *model.DPTask, err error) {
+	var (
+		t     *model.Task
+		cond  *model.DPCondition
+		group = errgroup.Group{}
+	)
+	group.Go(func() error {
+		t, err = s.dao.TaskInfo(ctx, id)
+		return err
+	})
+	group.Go(func() error {
+		cond, err = s.dao.DPCondition(ctx, job)
+		return err
+	})
+	if err = group.Wait(); err != nil {
+		return
+	}
+	if t == nil {
+		return
+	}
+	res = new(model.DPTask)
+	res.Task = *t
+	p := new(model.DPParams)
+	if cond != nil {
+		if err = json.Unmarshal([]byte(cond.Condition), &p); err != nil {
+			return
+		}
+	}
+	if len(p.VipExpires) == 0 {
+		p.VipExpires = make([]*model.VipExpire, 0)
+	}
+	if len(p.Attentions) == 0 {
+		p.Attentions = make([]*model.SelfAttention, 0)
+	}
+	if len(p.ActivePeriods) == 0 {
+		p.ActivePeriods = make([]*model.ActivePeriod, 0)
+	}
+	res.DPParams = *p
+	return
+}
+
+// AddDPTask add data platform task
+func (s *Service) AddDPTask(ctx context.Context, task *model.DPTask) (err error) {
+	var (
+		tasks   []model.DPTask
+		condTyp = task.Type
+	)
+	if len(task.DPParams.ActivePeriods) == 0 {
+		tasks = append(tasks, *task)
+	} else {
+		// 多个推送需要添加多条任务
+		for _, p := range task.DPParams.ActivePeriods {
+			var (
+				ptime time.Time
+				etime time.Time
+				t     = *task
+			)
+			if ptime, err = time.ParseInLocation(_timeLayout, p.PushTime, time.Local); err != nil {
+				return
+			}
+			if etime, err = time.ParseInLocation(_timeLayout, p.ExpireTime, time.Local); err != nil {
+				return
+			}
+			t.Job = strconv.FormatInt(pushmdl.JobName(ptime.Unix(), t.Summary, t.LinkValue, t.Group), 10)
+			t.PushTime = ptime
+			t.ExpireTime = etime
+			t.ActivePeriod = p.Period
+			tasks = append(tasks, t)
+		}
+	}
+	for _, t := range tasks {
+		go func(t model.DPTask) {
+			var (
+				id int64
+				e  error
+			)
+			for i := 0; i < _retry; i++ {
+				if id, e = s.dao.AddTask(context.Background(), &t.Task); err == nil {
+					break
+				}
+				time.Sleep(10 * time.Millisecond)
+			}
+			if e != nil {
+				log.Error("s.AddDPTask(%+v) add task error(%v)", t.Task, e)
+				return
+			}
+			t.DPParams.LevelStr = pushmdl.JoinInts(t.DPParams.Level)
+			t.DPParams.AreaStr = pushmdl.JoinInts(t.DPParams.Area)
+			t.DPParams.PlatformStr = pushmdl.JoinInts(t.DPParams.Platforms)
+			t.DPParams.LikeStr = pushmdl.JoinInts(t.DPParams.Like)
+			t.DPParams.ChannelStr = strings.Join(t.DPParams.Channel, ",")
+			params, _ := json.Marshal(t.DPParams)
+			cond := &model.DPCondition{
+				Task:      id,
+				Job:       t.Job,
+				Type:      condTyp,
+				Condition: string(params),
+				SQL:       s.parseQuery(task.Type, &t.DPParams),
+				Status:    pushmdl.DpCondStatusPending,
+			}
+			for i := 0; i < _retry; i++ {
+				if _, e = s.dao.AddDPCondition(context.Background(), cond); e == nil {
+					break
+				}
+				time.Sleep(10 * time.Millisecond)
+			}
+			if e != nil {
+				log.Error("s.AddDPTask(%+v) add condition error(%v)", cond, e)
+			}
+		}(t)
+	}
+	return
+}
+
+// 把查询结构体解析成sql
+func (s *Service) parseQuery(typ int, p *model.DPParams) (sql string) {
+	log.Info("data platform parse query start(%+v)", p)
+	if typ != pushmdl.TaskTypeDataPlatformToken && typ != pushmdl.TaskTypeDataPlatformMid {
+		log.Error("data platform parse query task type(%d) error", typ)
+		return
+	}
+	logDate := time.Now().Add(-24 * time.Hour).Format("20060102")
+	switch typ {
+	case pushmdl.TaskTypeDataPlatformToken:
+		sql = fmt.Sprintf("select platform_id,device_token from basic.dws_push_buvid where log_date='%s' ", logDate)
+		if p.UserActiveDay > 0 {
+			sql += fmt.Sprintf(" and is_visit_%d=1 ", p.UserActiveDay)
+		}
+		if p.UserNewDay > 0 {
+			sql += fmt.Sprintf(" and is_new_%d=1 ", p.UserNewDay)
+		}
+		if p.UserSilentDay > 0 {
+			sql += fmt.Sprintf(" and is_visit_%d=0 ", p.UserSilentDay)
+		}
+	case pushmdl.TaskTypeDataPlatformMid:
+		if len(p.Attentions) > 0 {
+			sql = fmt.Sprintf("select t1.platform_id,t1.device_token from (select mid,platform_id,device_token from basic.dws_push_mid where log_date='%s' ", logDate)
+		} else {
+			sql = fmt.Sprintf("select platform_id,device_token from basic.dws_push_mid where log_date='%s' ", logDate)
+		}
+		if p.Sex == _sexMale || p.Sex == _sexFemale {
+			sql += fmt.Sprintf(" and final_sex=%d ", p.Sex)
+		}
+		// 年龄段。0:0-17, 1:18-24, 2:25-30, 3:31+, 9:全部
+		if p.Age >= 0 && p.Age <= 3 {
+			sql += fmt.Sprintf(" and final_age_range=%d ", p.Age)
+		}
+		if len(p.Level) > 0 {
+			sql += fmt.Sprintf(" and level in (%s) ", pushmdl.JoinInts(p.Level))
+		}
+		// up主 0: 不是 1:是 2:全部
+		if p.IsUp == 0 || p.IsUp == 1 {
+			sql += fmt.Sprintf(" and is_up=%d ", p.IsUp)
+		}
+		// 正式会员 0:不是   1:是  2：全部
+		if p.IsFormalMember == 0 || p.IsFormalMember == 1 {
+			sql += fmt.Sprintf(" and is_formal_member=%d ", p.IsFormalMember)
+		}
+		if len(p.VipExpires) > 0 {
+			var vips []string
+			for _, v := range p.VipExpires {
+				if v.Begin == "" && v.End == "" {
+					continue
+				}
+				if v.Begin == "" {
+					vips = append(vips, fmt.Sprintf(" vip_overdue_time <= '%s'", v.End))
+					continue
+				}
+				if v.End == "" {
+					vips = append(vips, fmt.Sprintf(" vip_overdue_time >= '%s'", v.Begin))
+					continue
+				}
+				vips = append(vips, fmt.Sprintf(" vip_overdue_time between '%s' and '%s' ", v.Begin, v.End))
+			}
+			if len(vips) > 0 {
+				sql += fmt.Sprintf(" and (%s) ", strings.Join(vips, "or"))
+			}
+		}
+	}
+	if p.PlatformStr != "" && p.PlatformStr != _platformAll {
+		for _, v := range p.Platforms {
+			if v == pushmdl.PlatformAndroid {
+				for _, i := range pushmdl.Platforms {
+					if i == pushmdl.PlatformIPhone || i == pushmdl.PlatformIPad {
+						continue
+					}
+					p.Platforms = append(p.Platforms, i)
+				}
+				break
+			}
+		}
+		sql += fmt.Sprintf(" and platform_id in (%s) ", pushmdl.JoinInts(p.Platforms))
+	}
+	if len(p.Channel) > 0 {
+		var brands []string
+		for _, v := range p.Channel {
+			if v == "" {
+				continue
+			}
+			brands = append(brands, "'"+v+"'")
+		}
+		if len(brands) > 0 {
+			sql += fmt.Sprintf(" and device_brand in (%s) ", strings.Join(brands, ","))
+		}
+	}
+	if len(p.Area) > 0 {
+		var countries, provinces []string
+		for _, v := range p.Area {
+			if areas[v] == "" {
+				continue
+			}
+			if v == _countryForeign || v == _countryChina {
+				countries = append(countries, "'"+areas[v]+"'")
+			} else {
+				provinces = append(provinces, "'"+areas[v]+"'")
+			}
+		}
+		if len(countries) > 0 {
+			sql += fmt.Sprintf(" and country in(%s) ", strings.Join(countries, ","))
+		}
+		if len(provinces) > 0 {
+			sql += fmt.Sprintf(" and province in(%s) ", strings.Join(provinces, ","))
+		}
+	}
+	if len(p.Like) > 0 {
+		var likes []string
+		for _, id := range p.Like {
+			if s.partitions[id] == "" {
+				continue
+			}
+			likes = append(likes, "'"+s.partitions[id]+"'")
+		}
+		if len(likes) > 0 {
+			sql += fmt.Sprintf(" and like_tid in(%s) ", strings.Join(likes, ","))
+		}
+	}
+	if p.ActivePeriod > 0 {
+		sql += fmt.Sprintf(" and active_hour_period=%d ", p.ActivePeriod)
+	}
+	if typ == pushmdl.TaskTypeDataPlatformMid && len(p.Attentions) > 0 {
+		var attentions []string
+		if p.AttentionsType == _attentionTypeUnion {
+			// 并集
+			sql += ") t1 join ("
+			for _, v := range p.Attentions {
+				s := fmt.Sprintf("select mid from basic.dwd_oid_mid_info where log_date='%s' and follow_type=%d ", logDate, v.Type)
+				if v.Include != "" {
+					s += fmt.Sprintf(" and name='%s' ", strings.Replace(v.Include, "'", "\\'", -1))
+				}
+				if v.Exclude != "" {
+					s += fmt.Sprintf("and name!='%s'", strings.Replace(v.Exclude, "'", "\\'", -1))
+				}
+				attentions = append(attentions, s)
+			}
+			sql += strings.Join(attentions, " union ") + ") t2 on t1.mid=t2.mid"
+		} else {
+			// 交集
+			sql += ") t1 join "
+			for i, v := range p.Attentions {
+				s := fmt.Sprintf("(select mid from basic.dwd_oid_mid_info where log_date='%s' and follow_type=%d ", logDate, v.Type)
+				if v.Include != "" {
+					s += fmt.Sprintf(" and name='%s' ", strings.Replace(v.Include, "'", "\\'", -1))
+				}
+				if v.Exclude != "" {
+					s += fmt.Sprintf("and name!='%s'", strings.Replace(v.Exclude, "'", "\\'", -1))
+				}
+				s += fmt.Sprintf(") t%d on t1.mid=t%d.mid ", i+5, i+5)
+				attentions = append(attentions, s)
+			}
+			sql += strings.Join(attentions, " join ")
+		}
+	}
+	log.Info("data platform parse query end(%s)", sql)
+	return
+}
+
+// UpdateDpCondtionStatus .
+func (s *Service) UpdateDpCondtionStatus(ctx context.Context, job string, status int) (err error) {
+	s.dao.UpdateDpCondtionStatus(ctx, job, status)
+	return
+}
+
+type checkDpDataRes struct {
+	Code        int    `json:"code"`
+	Msg         string `json:"msg"`
+	ProjectID   int    `json:"projectId"`
+	ProjectName string `json:"projectName"`
+}
+
+// CheckDpData check whether data platform data is ready
+func (s *Service) CheckDpData(ctx context.Context) (err error) {
+	group := errgroup.Group{}
+	for id := range dpProjects {
+		id := id
+		now := time.Now()
+		bts := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix() * 1000 // 13位时间戳
+		ets := time.Date(now.Year(), now.Month(), now.Day(), 23, 59, 59, 0, time.Local).Unix() * 1000
+		params := dpparams(s.c.DPClient.Key)
+		params.Set("runStartTime", strconv.FormatInt(bts, 10))
+		params.Set("runEndTime", strconv.FormatInt(ets, 10))
+		group.Go(func() (err error) {
+			u := fmt.Sprintf(checkDpDataURL, id)
+			if enc := dpsign(s.c.DPClient.Secret, params); enc != "" {
+				u = u + "?" + enc
+			}
+			req, err := http.NewRequest(http.MethodGet, u, nil)
+			if err != nil {
+				log.Error("CheckDpData url(%s) error(%v)", u, err)
+				return
+			}
+			res := new(checkDpDataRes)
+			if err = s.dpClient.Do(ctx, req, res); err != nil {
+				log.Error("CheckDpData url(%s) error(%v)", u+"?"+params.Encode(), err)
+				return
+			}
+			if res.Code != http.StatusOK {
+				err = ecode.PushAdminDPNoDataErr
+			}
+			log.Info("check data platform url(%s) param(%s) res(%+v)", u, params.Encode(), res)
+			return
+		})
+	}
+	err = group.Wait()
+	return
+}
+
+// dpsign calc appkey and appsecret sign.
+func dpsign(secret string, params url.Values) (query string) {
+	tmp := params.Encode()
+	signTmp := dpencode(params)
+	if strings.IndexByte(tmp, '+') > -1 {
+		tmp = strings.Replace(tmp, "+", "%20", -1)
+	}
+	var b bytes.Buffer
+	b.WriteString(secret)
+	b.WriteString(signTmp)
+	b.WriteString(secret)
+	mh := md5.Sum(b.Bytes())
+	var qb bytes.Buffer
+	qb.WriteString(tmp)
+	qb.WriteString("&sign=")
+	qb.WriteString(strings.ToUpper(hex.EncodeToString(mh[:])))
+	query = qb.String()
+	return
+}
+
+func dpparams(appkey string) url.Values {
+	params := url.Values{}
+	params.Set("appKey", appkey)
+	params.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	params.Set("version", "1.0")
+	params.Set("signMethod", "md5")
+	return params
+}
+
+var dpSignParams = []string{"appKey", "timestamp", "version"}
+
+// encode data platform encodes the values into ``URL encoded'' form
+// ("bar=baz&foo=quux") sorted by key.
+func dpencode(v url.Values) string {
+	if v == nil {
+		return ""
+	}
+	var buf bytes.Buffer
+	keys := make([]string, 0, len(v))
+	for k := range v {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		found := false
+		for _, p := range dpSignParams {
+			if p == k {
+				found = true
+				break
+			}
+		}
+		if !found {
+			continue
+		}
+		vs := v[k]
+		prefix := k
+		for _, v := range vs {
+			buf.WriteString(prefix)
+			buf.WriteString(v)
+		}
+	}
+	return buf.String()
+}

app/admin/main/push/service/service.go

@@ -0,0 +1,107 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/admin/main/push/conf"
+	"go-common/app/admin/main/push/dao"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+
+	"github.com/jinzhu/gorm"
+)
+
+// Service biz service def.
+type Service struct {
+	c          *conf.Config
+	dao        *dao.Dao
+	DB         *gorm.DB
+	httpClient *bm.Client
+	dpClient   *bm.Client
+	partitions map[int]string
+}
+
+// New new a Service and return.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:          c,
+		dao:        dao.New(c),
+		httpClient: bm.NewClient(c.HTTPClient),
+		dpClient:   bm.NewClient(c.DPClient),
+		partitions: make(map[int]string),
+	}
+	s.DB = s.dao.DB.Scopes(func(db *gorm.DB) *gorm.DB {
+		return db.Where("dtime = ?", 0)
+	})
+	s.loadPartitions()
+	go s.loadPartitionsproc()
+	go s.cleanDiskFilesproc()
+	return s
+}
+
+func (s *Service) loadPartitions() (err error) {
+	m, err := s.dao.Partitions(context.Background())
+	if err != nil {
+		return
+	}
+	if len(m) > 0 {
+		s.partitions = m
+	}
+	return
+}
+
+func (s *Service) loadPartitionsproc() {
+	for {
+		if err := s.loadPartitions(); err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		time.Sleep(time.Minute)
+	}
+}
+
+func (s *Service) cleanDiskFilesproc() {
+	for {
+		fs, err := ioutil.ReadDir(conf.Conf.Cfg.MountDir)
+		if err != nil {
+			log.Error("s.cleanFilesproc() read dir error(%v)", err)
+			time.Sleep(time.Minute)
+			continue
+		}
+		divDate := time.Now().Add(-time.Duration(24*conf.Conf.Cfg.DiskFileExpireDay) * time.Hour).Format("20060102")
+		div, _ := strconv.ParseInt(divDate, 10, 64)
+		for _, f := range fs {
+			if !f.IsDir() {
+				continue
+			}
+			d, _ := strconv.ParseInt(f.Name(), 10, 64)
+			if d < div {
+				dir := fmt.Sprintf("%s/%s", strings.TrimSuffix(conf.Conf.Cfg.MountDir, "/"), f.Name())
+				if err = os.RemoveAll(dir); err != nil {
+					log.Error("s.cleanFilesproc() remove dir(%s) error(%v)", dir, err)
+					time.Sleep(time.Minute)
+				}
+			}
+		}
+		time.Sleep(time.Hour)
+	}
+}
+
+// Ping check dao health.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Wait wait all closed.
+func (s *Service) Wait() {}
+
+// Close close all dao.
+func (s *Service) Close() {
+	s.dao.Close()
+}

app/admin/main/push/service/service_test.go

@@ -0,0 +1,132 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"go-common/app/admin/main/push/conf"
+	"go-common/app/admin/main/push/model"
+	pushmdl "go-common/app/service/main/push/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var svr *Service
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/push-admin-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	svr = New(conf.Conf)
+	time.Sleep(time.Second)
+}
+
+func WithService(f func(s *Service)) func() {
+	return func() {
+		Reset(func() {})
+		f(svr)
+	}
+}
+
+func Test_Service(t *testing.T) {
+	Convey("service test", t, WithService(func(s *Service) {
+		s.Wait()
+		s.Close()
+	}))
+}
+
+func Test_AddTaskAll(t *testing.T) {
+	Convey("get last report id", t, WithService(func(s *Service) {
+		var maxID int64
+		err := s.dao.DB.Raw("SELECT MAX(id) FROM push_reports").Row().Scan(&maxID)
+		So(err, ShouldBeNil)
+		t.Logf("maxid(%d)", maxID)
+	}))
+
+	Convey("get tokens", t, WithService(func(s *Service) {
+		sqlStr := fmt.Sprintf("SELECT platform_id,device_token FROM push_reports WHERE id>%d and id<=%d and app_id=%d and dtime=0 and notify_switch=1", 1, 100, 1)
+		rows, err := s.dao.DB.Raw(sqlStr).Rows()
+		So(err, ShouldBeNil)
+		var tokens []string
+		for rows.Next() {
+			var (
+				platformID int
+				token      string
+			)
+			err = rows.Scan(&platformID, &token)
+			So(err, ShouldBeNil)
+			tokens = append(tokens, fmt.Sprintf("%d\t%s", platformID, token))
+		}
+		t.Logf("tokens(%d)", len(tokens))
+		if len(tokens) > 0 {
+			t.Logf("token one (%s)", tokens[0])
+		}
+	}))
+}
+
+func Test_CheckUploadMid(t *testing.T) {
+	Convey("CheckUploadMid", t, WithService(func(s *Service) {
+		data := []byte("1\n2\n3")
+		err := s.CheckUploadMid(context.TODO(), data)
+		So(err, ShouldBeNil)
+		data = []byte("1\nabc\n3")
+		err = s.CheckUploadMid(context.TODO(), data)
+		So(err, ShouldNotBeNil)
+		t.Logf("check mid error(%v)", err)
+	}))
+}
+
+func Test_CheckUploadToken(t *testing.T) {
+	Convey("CheckUploadToken", t, WithService(func(s *Service) {
+		data := []byte("2	fdsahjfkdshaj\n3	hjkhjhjkhj")
+		err := s.CheckUploadToken(context.TODO(), data)
+		So(err, ShouldBeNil)
+
+		data = []byte("2	fdsahjfkdshaj\n3	hjkhjhjkhj\n4\n")
+		err = s.CheckUploadToken(context.TODO(), data)
+		So(err, ShouldNotBeNil)
+		t.Logf("check token error(%v)", err)
+	}))
+}
+
+func Test_parseQuery(t *testing.T) {
+	Convey("parse query", t, WithService(func(s *Service) {
+		str := `{"age":1,"sex":1,"is_up":0,"is_formal_member":1,"user_active_day":0,"user_new_day":0,"user_silentDay":0,"area":"2","level":"2,3,4","platforms":"1,2,3","like":"1,2,3","channel":"huawei,xiaomi","vip_expire":[{"begin":"2018-07-01 00:00:00","end":"2018-07-27 00:00:00"}],"self_attention":null,"self_attention_type":0,"active":null,"ActivePeriod":0}`
+		p := new(model.DPParams)
+		err := json.Unmarshal([]byte(str), p)
+		So(err, ShouldBeNil)
+
+		p.Area = pushmdl.SplitInts(p.AreaStr)
+		p.Level = pushmdl.SplitInts(p.LevelStr)
+		p.Platforms = pushmdl.SplitInts(p.PlatformStr)
+		p.Like = pushmdl.SplitInts(p.LikeStr)
+		p.Channel = strings.Split(p.ChannelStr, ",")
+		if p.VipExpireStr != "" {
+			err = json.Unmarshal([]byte(p.VipExpireStr), &p.VipExpires)
+			So(err, ShouldBeNil)
+		}
+		if p.AttentionStr != "" {
+			err = json.Unmarshal([]byte(p.AttentionStr), &p.Attentions)
+			So(err, ShouldBeNil)
+		}
+		if p.ActivePeriodStr != "" {
+			err = json.Unmarshal([]byte(p.ActivePeriodStr), &p.ActivePeriods)
+			So(err, ShouldBeNil)
+		}
+		sql := s.parseQuery(pushmdl.TaskTypeDataPlatformMid, p)
+		t.Logf("sql(%s)", sql)
+	}))
+}
+
+func TestCheckDpData(t *testing.T) {
+	Convey("test check data platform data", t, WithService(func(s *Service) {
+		err := s.CheckDpData(context.Background())
+		So(err, ShouldBeNil)
+	}))
+}

app/admin/main/push/service/task.go

@@ -0,0 +1,154 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"go-common/app/admin/main/push/model"
+	pushmdl "go-common/app/service/main/push/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_testPushURL  = "http://api.bilibili.co/x/internal/push-service/push"
+	_testTokenURL = "http://api.bilibili.co/x/internal/push-service/test/token"
+)
+
+// AddTask add task.
+func (s *Service) AddTask(c context.Context, task *model.Task) (err error) {
+	if err = s.DB.Create(task).Error; err != nil {
+		s.dao.SendWechat("推送后台新建任务失败，原因：写入DB失败")
+		log.Error("s.AddTask(%+v) error(%v)", task, err)
+	}
+	return
+}
+
+// TestPushMid test push by mid.
+func (s *Service) TestPushMid(c context.Context, mid string, task *model.Task) (err error) {
+	params := url.Values{}
+	params.Set("app_id", fmt.Sprintf("%d", task.AppID))
+	params.Set("platform", task.Platform)
+	params.Set("alert_title", task.Title)
+	params.Set("alert_body", task.Summary)
+	params.Set("link_type", strconv.Itoa(task.LinkType))
+	params.Set("link_value", task.LinkValue)
+	params.Set("expire_time", fmt.Sprintf("%d", task.ExpireTimeUnix))
+	params.Set("builds", task.Build)
+	params.Set("sound", strconv.Itoa(task.Sound))
+	params.Set("vibration", strconv.Itoa(task.Vibration))
+	params.Set("mid", mid)
+	params.Set("image_url", task.ImageURL)
+	var res struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+	}
+	if err = s.httpClient.Post(c, _testPushURL, "", params, &res); err != nil {
+		log.Error("s.TestPush(%s) httpClient.Get(%s,%v) error(%v)", mid, _testPushURL, params, err)
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		log.Error("测试推送失败 mid(%s) code(%d)", mid, res.Code)
+		err = ecode.Int(res.Code)
+	}
+	return
+}
+
+// TestPushToken test push by token.
+func (s *Service) TestPushToken(c context.Context, info *pushmdl.PushInfo, platform int, token string) (err error) {
+	params := url.Values{}
+	params.Set("app_id", fmt.Sprintf("%d", info.APPID))
+	params.Set("platform", strconv.Itoa(platform))
+	params.Set("alert_title", info.Title)
+	params.Set("alert_body", info.Summary)
+	params.Set("link_type", fmt.Sprintf("%d", info.LinkType))
+	params.Set("link_value", info.LinkValue)
+	params.Set("expire_time", fmt.Sprintf("%d", int64(info.ExpireTime)))
+	params.Set("sound", strconv.Itoa(info.Sound))
+	params.Set("vibration", strconv.Itoa(info.Vibration))
+	params.Set("pass_through", strconv.Itoa(info.PassThrough))
+	params.Set("token", token)
+	var res struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+	}
+	if err = s.httpClient.Post(c, _testTokenURL, "", params, &res); err != nil {
+		log.Error("s.TestToken(%s) httpClient.Get(%s) error(%v)", token, params.Encode(), err)
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		log.Error("测试推送失败 token(%s) code(%d)", token, res.Code)
+		err = ecode.Int(res.Code)
+	}
+	return
+}
+
+// 上传说明：
+// 前端是批量上传，会随机按内容长度切割文件进行分批上传，有可能会切断原始行内容
+// 如果想在上传的时候同步判断文件格式，产生错误时需要忽略首行和末行
+
+// CheckUploadMid checks uploaded mid validation.
+func (s *Service) CheckUploadMid(c context.Context, data []byte) (err error) {
+	var (
+		mid     int64
+		lineNum int
+		lines   = strings.Split(string(data), "\n")
+		total   = len(lines)
+	)
+	for _, v := range lines {
+		lineNum++
+		v = strings.Trim(v, " \r\t")
+		if v == "" {
+			continue
+		}
+		if mid, err = strconv.ParseInt(v, 10, 64); err != nil {
+			log.Error("CheckUploadMid data(%s) error(%v)", v, err)
+			return ecode.PushUploadInvalidErr
+		}
+		if mid <= 0 {
+			if lineNum == 1 || lineNum == total {
+				continue
+			}
+			log.Error("CheckUploadMid data(%s) error(%v)", v, err)
+			return ecode.PushUploadInvalidErr
+		}
+	}
+	return
+}
+
+// CheckUploadToken checks uploaded token validation.
+func (s *Service) CheckUploadToken(c context.Context, data []byte) (err error) {
+	var (
+		plat    int
+		lineNum int
+		lines   = strings.Split(string(data), "\n")
+		total   = len(lines)
+	)
+	for _, v := range lines {
+		lineNum++
+		if lineNum == 1 || lineNum == total {
+			continue
+		}
+		v = strings.Trim(v, " \r")
+		if v == "" {
+			continue
+		}
+		res := strings.Split(v, "\t")
+		if len(res) != 2 {
+			log.Error("CheckUploadToken data(%s)", v)
+			return ecode.PushUploadInvalidErr
+		}
+		if res[0] == "" || res[1] == "" {
+			log.Error("CheckUploadToken data(%s)", v)
+			return ecode.PushUploadInvalidErr
+		}
+		if plat, err = strconv.Atoi(res[0]); err != nil || plat <= 0 {
+			log.Error("CheckUploadToken data(%s) error(%v)", v, err)
+			return ecode.PushUploadInvalidErr
+		}
+	}
+	return
+}

app/admin/main/push/service/upload.go

@@ -0,0 +1,109 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// Upload add mids file.
+func (s *Service) Upload(c context.Context, dir, path string, data []byte) (err error) {
+	if _, err = os.Stat(dir); err != nil {
+		if !os.IsNotExist(err) {
+			log.Error("os.IsNotExist(%s) error(%v)", dir, err)
+			return
+		}
+		if err = os.MkdirAll(dir, 0777); err != nil {
+			log.Error("os.MkdirAll(%s) error(%v)", dir, err)
+			return
+		}
+	}
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		log.Error("s.Upload(%s) error(%v)", path, err)
+		return
+	}
+	defer f.Close()
+	if _, err = f.Write(data); err != nil {
+		log.Error("f.Write() error(%v)", err)
+	}
+	return
+}
+
+// Upimg upload image
+func (s *Service) Upimg(ctx context.Context, file multipart.File, header *multipart.FileHeader) (url string, err error) {
+	buf := new(bytes.Buffer)
+	w := multipart.NewWriter(buf)
+	dest, err := w.CreateFormFile("file", header.Filename)
+	if err != nil {
+		log.Error("Upimg CreateFormFile error(%v)", err)
+		return
+	}
+	if _, err = io.Copy(dest, file); err != nil {
+		log.Error("Upimg Copy error(%v)", err)
+		return
+	}
+	w.Close()
+	query := map[string]string{
+		"ts":     strconv.FormatInt(time.Now().Unix(), 10),
+		"appkey": s.c.HTTPClient.App.Key,
+	}
+	query["sign"] = sign(query, s.c.HTTPClient.App.Secret)
+	u := fmt.Sprintf("%s?ts=%s&appkey=%s&sign=%s", s.c.Cfg.UpimgURL, query["ts"], query["appkey"], query["sign"])
+	req, err := http.NewRequest(http.MethodPost, u, buf)
+	if err != nil {
+		log.Error("http.NewRequest(%s) error(%v)", u, err)
+		return
+	}
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	res := &struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+		Data    struct {
+			URL string `json:"url"`
+		} `json:"data"`
+	}{}
+	if err = s.httpClient.Do(context.TODO(), req, &res); err != nil {
+		log.Error("httpClient.Do() error(%v)", err)
+		return
+	}
+	if !ecode.Int(res.Code).Equal(ecode.OK) {
+		log.Error("upimg error code(%d)", res.Code)
+		err = ecode.Int(res.Code)
+		return
+	}
+	url = strings.Replace(res.Data.URL, "http://", "https://", 1)
+	return
+}
+
+func sign(params map[string]string, secret string) string {
+	var keys []string
+	for k := range params {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	buf := bytes.Buffer{}
+	for _, k := range keys {
+		if buf.Len() > 0 {
+			buf.WriteByte('&')
+		}
+		buf.WriteString(url.QueryEscape(k) + "=")
+		buf.WriteString(url.QueryEscape(params[k]))
+	}
+	h := md5.New()
+	io.WriteString(h, buf.String()+secret)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}