Create & Init Project...

app/job/openplatform/open-market/BUILD

@@ -0,0 +1,22 @@
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/openplatform/open-market/cmd:all-srcs",
+        "//app/job/openplatform/open-market/conf:all-srcs",
+        "//app/job/openplatform/open-market/dao:all-srcs",
+        "//app/job/openplatform/open-market/http:all-srcs",
+        "//app/job/openplatform/open-market/model:all-srcs",
+        "//app/job/openplatform/open-market/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

app/job/openplatform/open-market/CHANGELOG.md

@@ -0,0 +1,7 @@
+#### open-market-job
+
+##### v1.0.0
+> 项目初始化合入go-common,运营元数据整理，其他相关功能陆续搬运至此项目
+
+##### v1.0.1
+> 订单es环境由测试环境迁移至正式环境

app/job/openplatform/open-market/CONTRIBUTORS.md

@@ -0,0 +1,11 @@
+# Owner
+changxuanran
+liuyan02
+qiuliang
+
+# Author
+changxuanran
+liuyan02
+
+# Reviewer
+qiuliang

app/job/openplatform/open-market/OWNERS

@@ -0,0 +1,16 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- changxuanran
+- liuyan02
+- qiuliang
+labels:
+- job
+- job/openplatform/open-market
+- openplatform
+options:
+  no_parent_owners: true
+reviewers:
+- changxuanran
+- liuyan02
+- qiuliang

app/job/openplatform/open-market/README.md

@@ -0,0 +1,10 @@
+#### open-market-job
+
+##### 项目简介
+> 1.开放平台运营相关数据整理
+
+##### 编译环境
+> 请只用golang v1.8.x以上版本编译执行。
+
+##### 依赖包
+> 1.公共包go-common

app/job/openplatform/open-market/cmd/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["open-market-job-test.toml"],
+    importpath = "go-common/app/job/openplatform/open-market/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/openplatform/open-market/conf:go_default_library",
+        "//app/job/openplatform/open-market/http:go_default_library",
+        "//app/job/openplatform/open-market/service:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/open-market/cmd/main.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/job/openplatform/open-market/conf"
+	"go-common/app/job/openplatform/open-market/http"
+	"go-common/app/job/openplatform/open-market/service"
+	"go-common/library/log"
+)
+
+var (
+	svr *service.Service
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Xlog)
+	defer log.Close()
+	// service init
+	svr = service.New(conf.Conf)
+	http.Init(conf.Conf, svr)
+	log.Info("open-market-job start")
+	// init signal
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("open-market-job get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
+			log.Info("open-market-job exit")
+			if err := svr.Close(); err != nil {
+				log.Error("srv close consumer error(%v)", err)
+			}
+			time.Sleep(2 * time.Second)
+			return
+		case syscall.SIGHUP:
+		// TODO reload
+		default:
+			return
+		}
+	}
+}

app/job/openplatform/open-market/cmd/open-market-job-test.toml

@@ -0,0 +1,49 @@
+[db.ticketDB]
+    addr = "172.22.34.101:3310"
+    dsn = "ticket:i9HXWAvzWiqPxMxlfsQ8DRqYydjf3pYa@tcp(172.22.34.101:3310)/open_ticket?timeout=5s&readTimeout=30s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 5
+    idle = 2
+    queryTimeout = "5000ms"
+    execTimeout = "100ms"
+    tranTimeout = "200ms"
+    [db.ticketDB.breaker]
+        window = "3s"
+        sleep = "1000ms"
+        bucket = 10
+        ratio = 0.5
+        request = 500
+
+[httpClient]
+    key = "654af11b5df0c9d3"
+    secret = "a7512b8b243b82f4bdb72cf2824b3f8e"
+    dial = "500ms"
+    timeout = "1s"
+    keepAlive = "60s"
+    timer = 10
+    [httpClient.breaker]
+        window  = "3s"
+        sleep   = "100ms"
+        bucket  = 10
+        ratio   = 0.5
+        request = 100
+
+
+[bm]
+    [bm.inner]
+    addr = "0.0.0.0:7081"
+    timeout = "1s"
+
+[elasticSearch]
+    addr = ["http://172.16.38.66:9201"]
+    check = "10s"
+    timeout = "3s"
+
+[elasticSearchUgc]
+    addr = ["http://172.16.38.66:9200"]
+    check = "10s"
+    timeout = "3s"
+
+[berserker]
+    appkey = "7a374e9a8fd3938ac682cc6cb07855e8"
+    secret = "ad994843be00732bd6e65f9f95887bac"
+    url = "http://berserker.bilibili.co/avenger/api/77/query"

app/job/openplatform/open-market/conf/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/openplatform/open-market/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/open-market/conf/conf.go

@@ -0,0 +1,121 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+// Conf global variable.
+var (
+	Conf     = &Config{}
+	client   *conf.Client
+	confPath string
+)
+
+// Config struct of conf.
+type Config struct {
+	// base
+	// Xlog xlog conf
+	Xlog *log.Config
+	// DB multiple db conf
+	DB *DB
+	// HTTPClient httpClient conf
+	HTTPClient *bm.ClientConfig
+	//ES
+	ElasticSearch    *ElasticSearch
+	ElasticSearchUgc *ElasticSearch
+	//comment
+	Comment *Comment
+	BM      *HTTPServers
+	//berserker
+	Berserker *Berserker
+}
+
+// HTTPServers bm inner config
+type HTTPServers struct {
+	Inner *bm.ServerConfig
+}
+
+// Berserker api conf
+type Berserker struct {
+	Appkey string
+	Secret string
+	URL    string
+}
+
+// ElasticSearch elasticSearch.
+type ElasticSearch struct {
+	Addr    []string
+	Check   xtime.Duration
+	Timeout string
+}
+
+// Comment config with url and type
+type Comment struct {
+	URL  string
+	Type int
+}
+
+// DB config
+type DB struct {
+	TicketDB *sql.Config
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf.
+func Init() (err error) {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Value("open-market-job.toml"); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/openplatform/open-market/dao/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "berserker.go",
+        "dao.go",
+        "market_es.go",
+        "market_mysql.go",
+    ],
+    importpath = "go-common/app/job/openplatform/open-market/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/openplatform/open-market/conf:go_default_library",
+        "//app/job/openplatform/open-market/model:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/gopkg.in/olivere/elastic.v5:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/open-market/dao/berserker.go

@@ -0,0 +1,148 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+
+	"go-common/app/job/openplatform/open-market/model"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+
+	xhttp "net/http"
+
+	pkgerr "github.com/pkg/errors"
+)
+
+const (
+	_userAgent = "User-Agent"
+	_queryJSON = `{"select":[{"name":"item_id"},{"name":"pv"},{"name":"uv"},{"name":"days_before"}],"where":{"item_id":{"in":[%d]}},"page":{"limit":30,"skip":0}}`
+)
+
+var (
+	signParams = []string{"appKey", "timestamp", "version"}
+)
+
+// QueryPUVCount get puv info from berserker
+func (d *Dao) QueryPUVCount(c context.Context, itemID int32) (pv map[int32]int64, uv map[int32]int64, err error) {
+	pv = make(map[int32]int64)
+	uv = make(map[int32]int64)
+	v := make(url.Values, 8)
+	v.Set("appKey", d.c.Berserker.Appkey)
+	v.Set("signMethod", "md5")
+	v.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	v.Set("version", "1.0")
+	query := fmt.Sprintf(_queryJSON, itemID)
+	v.Set("query", query)
+	var res struct {
+		Code   int               `json:"code"`
+		Result []model.PUVResult `json:"result"`
+	}
+
+	if err = d.doHTTPRequest(c, d.c.Berserker.URL, "", v, &res); err != nil {
+		log.Error(d.c.Berserker.URL+"?"+v.Encode(), err)
+		return
+	}
+
+	if res.Code == 200 && len(res.Result) > 0 {
+		for _, v := range res.Result {
+			pv[v.DaysBefore] = v.PV
+			uv[v.DaysBefore] = v.UV
+		}
+		return
+	}
+	return
+}
+
+// doHttpRequest make a http request for data platform api
+func (d *Dao) doHTTPRequest(c context.Context, uri, ip string, params url.Values, res interface{}) (err error) {
+	enc, err := d.sign(params)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "uri:%s,params:%v", uri, params)
+		return
+	}
+	if enc != "" {
+		uri = uri + "?" + enc
+	}
+
+	req, err := xhttp.NewRequest(xhttp.MethodGet, uri, nil)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "method:%s,uri:%s", xhttp.MethodGet, uri)
+		return
+	}
+	req.Header.Set(_userAgent, "changxuanran@bilibili.com  "+env.AppID)
+	if err != nil {
+		return
+	}
+	return d.client.Do(c, req, res)
+}
+
+// Sign calc appkey and appsecret sign.
+func (d *Dao) sign(params url.Values) (query string, err error) {
+	tmp := params.Encode()
+	signTmp := d.encode(params)
+	if strings.IndexByte(tmp, '+') > -1 {
+		tmp = strings.Replace(tmp, "+", "%20", -1)
+	}
+	var b bytes.Buffer
+	b.WriteString(d.c.Berserker.Secret)
+	b.WriteString(signTmp)
+	b.WriteString(d.c.Berserker.Secret)
+	mh := md5.Sum(b.Bytes())
+	// query
+	var qb bytes.Buffer
+	qb.WriteString(tmp)
+	qb.WriteString("&sign=")
+	qb.WriteString(strings.ToUpper(hex.EncodeToString(mh[:])))
+	query = qb.String()
+	return
+}
+
+// Encode encodes the values into ``URL encoded'' form
+// ("bar=baz&foo=quux") sorted by key.
+func (d *Dao) encode(v url.Values) string {
+	if v == nil {
+		return ""
+	}
+	var buf bytes.Buffer
+	keys := make([]string, 0, len(v))
+	for k := range v {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		found := false
+		for _, p := range signParams {
+			if p == k {
+				found = true
+				break
+			}
+		}
+		if !found {
+			continue
+		}
+		vs := v[k]
+		prefix := k
+		for _, v := range vs {
+			buf.WriteString(prefix)
+			buf.WriteString(v)
+		}
+	}
+	return buf.String()
+}
+
+//func (d *Dao) reverse(s string) string {
+//	n := len(s)
+//	runes := make([]rune, n)
+//	for _, rune := range s {
+//		n--
+//		runes[n] = rune
+//	}
+//	return string(runes[n:])
+//}

app/job/openplatform/open-market/dao/dao.go

@@ -0,0 +1,88 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/openplatform/open-market/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+
+	elastic "gopkg.in/olivere/elastic.v5"
+)
+
+//Dao struct
+type Dao struct {
+	c *conf.Config
+	// http client
+	client *bm.Client
+	// db
+	ticketDB *sql.DB
+	//es
+	es    *elastic.Client
+	esUgc *elastic.Client
+}
+
+// New new a Dao and return.
+func New(c *conf.Config) (d *Dao) {
+	var (
+		err   error
+		es    *elastic.Client
+		esUgc *elastic.Client
+	)
+	d = &Dao{
+		c:        c,
+		client:   bm.NewClient(c.HTTPClient),
+		ticketDB: sql.NewMySQL(c.DB.TicketDB),
+	}
+	es, err = elastic.NewClient(
+		elastic.SetURL(c.ElasticSearch.Addr...),
+		elastic.SetSniff(false),
+		elastic.SetHealthcheckInterval(time.Duration(c.ElasticSearch.Check)),
+		elastic.SetErrorLog(&elog{}),
+		elastic.SetInfoLog(&ilog{}),
+	)
+	if err != nil {
+		panic(err)
+	}
+	esUgc, err = elastic.NewClient(
+		elastic.SetURL(c.ElasticSearchUgc.Addr...),
+		elastic.SetSniff(false),
+		elastic.SetHealthcheckInterval(time.Duration(c.ElasticSearch.Check)),
+		elastic.SetErrorLog(&elog{}),
+		elastic.SetInfoLog(&ilog{}),
+	)
+	if err != nil {
+		panic(err)
+	}
+	d.es = es
+	d.esUgc = esUgc
+	return d
+}
+
+// Ping ping health.
+func (d *Dao) Ping(c context.Context) (err error) {
+	return d.ticketDB.Ping(c)
+}
+
+// Close close.
+func (d *Dao) Close() (err error) {
+	if err = d.ticketDB.Close(); err != nil {
+		log.Error("dao.ticketDB.Close() error(%v)", err)
+	}
+	return
+}
+
+type ilog struct{}
+type elog struct{}
+
+// Printf printf.
+func (l *ilog) Printf(format string, v ...interface{}) {
+	log.Info(format, v...)
+}
+
+// Printf printf.
+func (l *elog) Printf(format string, v ...interface{}) {
+	log.Error(format, v...)
+}

app/job/openplatform/open-market/dao/market_es.go

@@ -0,0 +1,133 @@
+package dao
+
+import (
+	"context"
+	"gopkg.in/olivere/elastic.v5"
+	"strconv"
+	"time"
+
+	"go-common/app/job/openplatform/open-market/model"
+	"go-common/library/log"
+)
+
+var (
+	orderIndex   = "product_order"
+	orderType    = "product_order_info"
+	commentIndex = "open_mall"
+	commentType  = "ugc"
+	marketIndex  = "product_market"
+	marketType   = "product_market_info"
+)
+
+const (
+	_orderStatusComplete = 2
+	_dateFormat          = "2006-01-02"
+	_timeFormat          = "2006-01-02 15:04:05"
+)
+
+//OrderData fetch ordercount by project and days
+func (d *Dao) OrderData(c context.Context, projectID int32, startTimeUnix int64) (orderData map[int32]int64, err error) {
+	var (
+		searchResult *elastic.SearchResult
+		startTime    string
+		firstTime    string
+		firstDay     time.Time
+		startDay     time.Time
+		daysBefore   = -1
+	)
+	orderData = make(map[int32]int64)
+	startTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Format(_dateFormat)
+	firstTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Add(time.Hour * 24 * -30).Format(_dateFormat)
+	startDay, _ = time.Parse(_dateFormat, startTime)
+	firstDay, _ = time.Parse(_dateFormat, firstTime)
+	for {
+		daysBefore++
+		startDay = startDay.Add(time.Hour * -24)
+		if !(startDay.Before(firstDay)) {
+			rangeQuery := elastic.NewRangeQuery("pay_time")
+			rangeQuery.Gte(startDay.Format(_timeFormat))
+			rangeQuery.Lte(startDay.Add(time.Hour * 24).Format(_timeFormat))
+			query := elastic.NewBoolQuery()
+			query.Must(elastic.NewMatchQuery("project_id", projectID))
+			query.Must(elastic.NewMatchQuery("status", _orderStatusComplete))
+			query.Must(elastic.NewExistsQuery("pay_time"))
+			query.Must(rangeQuery)
+			searchResult, err = d.es.Search().
+				Index(orderIndex).Type(orderType).
+				Query(query).
+				Size(0).
+				Timeout(d.c.ElasticSearch.Timeout).
+				Do(c)
+			orderData[int32(daysBefore)] = searchResult.TotalHits()
+			continue
+		}
+		break
+	}
+	return
+}
+
+// CommentData get comment info from ugc es
+func (d *Dao) CommentData(c context.Context, projectID int32, startTimeUnix int64) (commentData map[int32]int64, err error) {
+	var (
+		searchResult *elastic.SearchResult
+		startTime    string
+		firstTime    string
+		firstDay     time.Time
+		startDay     time.Time
+		daysBefore   = -1
+	)
+	commentData = make(map[int32]int64)
+	startTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Format(_dateFormat)
+	firstTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Add(time.Hour * 24 * -30).Format(_dateFormat)
+	startDay, _ = time.Parse(_dateFormat, startTime)
+	firstDay, _ = time.Parse(_dateFormat, firstTime)
+	for {
+		daysBefore++
+		startDay = startDay.Add(time.Hour * -24)
+		if !(startDay.Before(firstDay)) {
+			rangeQuery := elastic.NewRangeQuery("ctime")
+			rangeQuery.Gte(startDay.Unix() * 1000)
+			rangeQuery.Lte(startDay.Add(time.Hour*24).Unix() * 1000)
+			query := elastic.NewBoolQuery()
+			query.Must(elastic.NewMatchQuery("subjectId", projectID))
+			query.Must(elastic.NewMatchQuery("subjectType", 2))
+			query.Must(rangeQuery)
+			searchResult, err = d.esUgc.Search().
+				Index(commentIndex).Type(commentType).
+				Query(query).
+				Size(0).
+				Timeout(d.c.ElasticSearch.Timeout).
+				Do(c)
+			commentData[int32(daysBefore)] = searchResult.TotalHits()
+			continue
+		}
+		break
+	}
+	return
+}
+
+//SaveData save result to es
+func (d *Dao) SaveData(c context.Context, project *model.Project) (err error) {
+	exists, err := d.es.IndexExists(marketIndex).Do(context.Background())
+	if err != nil {
+		log.Error("check if index exists error (%v)", err)
+		return
+	}
+	if !exists {
+		if _, err = d.es.CreateIndex(marketIndex).Do(c); err != nil {
+			log.Error("index name(%s) create err(%v)", marketIndex, err)
+			return
+		}
+	}
+	_, err = d.es.Index().
+		Index(marketIndex).
+		Type(marketType).
+		Id(strconv.Itoa(int(project.ID))).
+		BodyJson(project).
+		Refresh("true").
+		Do(c)
+	if err != nil {
+		log.Error("put es [%d] err(%v)", project.ID, err)
+	}
+	return
+}

app/job/openplatform/open-market/dao/market_mysql.go

@@ -0,0 +1,137 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/openplatform/open-market/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_fetchProjectList = "SELECT `id`,`name`,`start_time`,`type` FROM `project` WHERE `status` = 1 AND `type` IN (1,3,4,5,6,8)"
+
+	//_fetchUserWishFirst = "SELECT `mtime` FROM `user_wish` WHERE `item_id` = ? ORDER BY `mtime` LIMIT 1"
+	_fetchUserWishByDay = "SELECT DISTINCT `id` FROM `user_wish` WHERE `item_id` = ? AND `mtime` BETWEEN ? AND ?"
+
+	//_fetchUserFavoriteFirst = "SELECT `mtime` FROM `user_favorite` WHERE `item_id` = ? AND `status` = 1 AND `type` = 1 ORDER BY `mtime` LIMIT 1"
+	_fetchUserFavoriteByDay = "SELECT DISTINCT `id` FROM `user_favorite` WHERE `item_id` = ? AND `status` = 1 AND `type` = 1  AND `mtime` BETWEEN ? AND ? "
+
+	_fetchStockIfSoldOut = "SELECT `sku_id` FROM `sku_stock` WHERE `item_id` = ? AND `stock` <> 0  LIMIT 1"
+	//_fetchStockSoldOutDay = "SELECT `mtime` FROM `sku_stock` WHERE `item_id` = ? AND `stock` = 0 ORDER BY `mtime` DESC LIMIT 1"
+)
+
+//FetchProject query project list from mysql
+func (d *Dao) FetchProject(c context.Context) (projectList []*model.Project, err error) {
+	var rows *sql.Rows
+	if rows, err = d.ticketDB.Query(c, _fetchProjectList); err != nil {
+		log.Error("d._fetchProjectListSQL.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		project := new(model.Project)
+		if err = rows.Scan(&project.ID, &project.Name, &project.StartTime, &project.Type); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			projectList = nil
+			return
+		}
+		//if project.ID == 11644 || project.ID == 11678 || project.ID == 11643 || project.ID == 11650 {
+		//	continue
+		//}
+		if soldOut := d.CheckStock(context.TODO(), project.ID); soldOut {
+			continue
+		}
+		projectList = append(projectList, project)
+	}
+	err = rows.Err()
+	return
+}
+
+//CheckStock if a project sold out,return date
+func (d *Dao) CheckStock(c context.Context, projectID int32) (soldOut bool) {
+	row := d.ticketDB.QueryRow(c, _fetchStockIfSoldOut, projectID)
+	var skuID int
+	if err := row.Scan(&skuID); err != nil {
+		if err == sql.ErrNoRows {
+			return true
+		}
+		log.Error("row.Scan() error(%v)", err)
+	}
+	return
+}
+
+//WishData fetch wishcount by project and date
+func (d *Dao) WishData(c context.Context, projectID int32, startTimeUnix int64) (wishData map[int32]int64, err error) {
+	var (
+		startTime  string
+		firstTime  string
+		firstDay   time.Time
+		startDay   time.Time
+		daysBefore = -1
+	)
+	wishData = make(map[int32]int64)
+	startTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Format(_dateFormat)
+	firstTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Add(time.Hour * 24 * -30).Format(_dateFormat)
+	startDay, _ = time.Parse(_dateFormat, startTime)
+	firstDay, _ = time.Parse(_dateFormat, firstTime)
+	for {
+		daysBefore++
+		startDay = startDay.Add(time.Hour * -24)
+		if !(startDay.Before(firstDay)) {
+			var rows *sql.Rows
+			count := 0
+			if rows, err = d.ticketDB.Query(c, _fetchUserWishByDay, projectID, startDay.Format(_dateFormat), startDay.Add(time.Hour*24).Format(_dateFormat)); err != nil {
+				log.Error("wish count query error(%v)", err)
+				return
+			}
+			for rows.Next() {
+				count++
+			}
+			err = rows.Err()
+			wishData[int32(daysBefore)] = int64(count)
+			rows.Close()
+			continue
+		}
+		break
+	}
+	return
+}
+
+//FavoriteData fetch favoritecount by project and date
+func (d *Dao) FavoriteData(c context.Context, projectID int32, startTimeUnix int64) (favoriteData map[int32]int64, err error) {
+	var (
+		startTime  string
+		firstTime  string
+		firstDay   time.Time
+		startDay   time.Time
+		daysBefore = -1
+	)
+	favoriteData = make(map[int32]int64)
+	startTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Format(_dateFormat)
+	firstTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Add(time.Hour * 24 * -30).Format(_dateFormat)
+	startDay, _ = time.Parse(_dateFormat, startTime)
+	firstDay, _ = time.Parse(_dateFormat, firstTime)
+	for {
+		daysBefore++
+		startDay = startDay.Add(time.Hour * -24)
+		if !(startDay.Before(firstDay)) {
+			var rows *sql.Rows
+			count := 0
+			if rows, err = d.ticketDB.Query(c, _fetchUserFavoriteByDay, projectID, startDay.Format(_dateFormat), startDay.Add(time.Hour*24).Format(_dateFormat)); err != nil {
+				log.Error("wish count query error(%v)", err)
+				return
+			}
+			for rows.Next() {
+				count++
+			}
+			err = rows.Err()
+			favoriteData[int32(daysBefore)] = int64(count)
+			rows.Close()
+			continue
+		}
+		break
+	}
+	return
+}

app/job/openplatform/open-market/http/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/openplatform/open-market/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/openplatform/open-market/conf:go_default_library",
+        "//app/job/openplatform/open-market/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/open-market/http/http.go

@@ -0,0 +1,39 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/openplatform/open-market/conf"
+	"go-common/app/job/openplatform/open-market/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var srv *service.Service
+
+// Init init http sever instance.
+func Init(c *conf.Config, s *service.Service) {
+	srv = s
+	// init external router
+	engineIn := bm.DefaultServer(c.BM.Inner)
+	innerRouter(engineIn)
+	// init Inner server
+	if err := engineIn.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+// innerRouter init inner router.
+func innerRouter(e *bm.Engine) {
+	e.Ping(ping)
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {
+	err := srv.Ping(c)
+	if err != nil {
+		log.Error("app-job service ping error(%+v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}

app/job/openplatform/open-market/model/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["market.go"],
+    importpath = "go-common/app/job/openplatform/open-market/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/open-market/model/market.go

@@ -0,0 +1,86 @@
+package model
+
+//Project struct
+type Project struct {
+	ID           int32           `json:"id"`
+	Name         string          `json:"name"`
+	StartTime    int64           `json:"start_time"`
+	Type         int             `json:"type"`
+	PV           map[int32]int64 `json:"pv"`
+	UV           map[int32]int64 `json:"uv"`
+	SaleInfo     map[int32]int64 `json:"sale_info"`
+	WishInfo     map[int32]int64 `json:"wish_info"`
+	CommentInfo  map[int32]int64 `json:"comment_info"`
+	FavoriteInfo map[int32]int64 `json:"favorite_info"`
+}
+
+//OrderSkus struct
+type OrderSkus struct {
+	ProjectID   int    `json:"project_id"`
+	ProjectName string `json:"project_name"`
+	ScreenID    int    `json:"screen_id"`
+	ScreenName  string `json:"screen_name"`
+	SkuID       int    `json:"sku_id"`
+	Count       int    `json:"count"`
+	OriginPrice int    `json:"origin_price"`
+	Price       int    `json:"price"`
+	TicketType  string `json:"ticket_type"`
+	Desc        string `json:"desc"`
+}
+
+//Order struct
+type Order struct {
+	OrderID      int         `json:"order_id"`
+	Ctime        string      `json:"ctime"`
+	Mtime        string      `json:"mtime"`
+	OrderType    int         `json:"order_type"`
+	Source       string      `json:"source"`
+	PayMoney     int         `json:"pay_money"`
+	TotalMoney   int         `json:"total_money"`
+	ExpressFee   int         `json:"express_fee"`
+	UID          string      `json:"uid"`
+	PersonalID   string      `json:"personal_id"`
+	Tel          string      `json:"tel"`
+	Status       int         `json:"status"`
+	SubStatus    int         `json:"sub_status"`
+	RefundStatus int         `json:"refund_status"`
+	PayTime      string      `json:"pay_time"`
+	Skus         []OrderSkus `json:"skus"`
+	ProjectID    int         `json:"project_id"`
+	DistUserID   int         `json:"dist_user_id"`
+	DistType     int         `json:"dist_type"`
+	SendStatus   int         `json:"send_status"`
+}
+
+//Comment struct
+type Comment struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+	TTL     int    `json:"ttl"`
+	Data    *struct {
+		Page *struct {
+			Count int `json:"count"`
+			Num   int `json:"num"`
+			Size  int `json:"size"`
+		}
+		Replies []*struct {
+			Ctime  int64 `json:"ctime"`
+			Member *struct {
+				Mid   string `json:"mid"`
+				Uname string `json:"uname"`
+			} `json:"member"`
+		} `json:"replies"`
+	} `json:"data"`
+}
+
+// PUVResult get  responce
+type PUVResult struct {
+	PUV
+	DaysBefore int32 `json:"days_before"`
+}
+
+// PUV puv info
+type PUV struct {
+	PV int64 `json:"pv"`
+	UV int64 `json:"uv"`
+}

app/job/openplatform/open-market/service/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "market.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/openplatform/open-market/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/openplatform/open-market/conf:go_default_library",
+        "//app/job/openplatform/open-market/dao:go_default_library",
+        "//app/job/openplatform/open-market/model:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/open-market/service/market.go

@@ -0,0 +1,45 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/openplatform/open-market/model"
+	"go-common/library/log"
+)
+
+//fetch market data ,from db to es
+func (s *Service) marketProc() (err error) {
+	var projectList []*model.Project
+	if projectList, err = s.dao.FetchProject(context.TODO()); err != nil {
+		log.Error("pull project list error (%v)", err)
+		return
+	}
+	for _, project := range projectList {
+		var orderData, wishData, favoriteData, commentData, pvData, uvData map[int32]int64
+		if orderData, err = s.dao.OrderData(context.TODO(), project.ID, project.StartTime); err != nil {
+			log.Error("fetch project_order_data [%d] error (%v)", project.ID, err)
+		}
+		if commentData, err = s.dao.CommentData(context.TODO(), project.ID, project.StartTime); err != nil {
+			log.Error("fetch project_comment_data [%d] error (%v)", project.ID, err)
+		}
+		if wishData, err = s.dao.WishData(context.TODO(), project.ID, project.StartTime); err != nil {
+			log.Error("fetch project_wish_data [%d] error (%v)", project.ID, err)
+		}
+		if favoriteData, err = s.dao.FavoriteData(context.TODO(), project.ID, project.StartTime); err != nil {
+			log.Error("fetch project_favorite_data [%d] error (%v)", project.ID, err)
+		}
+		if pvData, uvData, err = s.dao.QueryPUVCount(context.TODO(), project.ID); err != nil {
+			log.Error("fetch project_puv_data [%d] error (%v)", project.ID, err)
+		}
+		project.PV = pvData
+		project.UV = uvData
+		project.SaleInfo = orderData
+		project.CommentInfo = commentData
+		project.WishInfo = wishData
+		project.FavoriteInfo = favoriteData
+		if err = s.dao.SaveData(context.TODO(), project); err != nil {
+			log.Error("save [%d]data to es  error (%v)", project.ID, err)
+		}
+	}
+	return
+}

app/job/openplatform/open-market/service/service.go

@@ -0,0 +1,44 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/openplatform/open-market/conf"
+	"go-common/app/job/openplatform/open-market/dao"
+)
+
+// Service struct of service.
+type Service struct {
+	c   *conf.Config
+	dao *dao.Dao
+}
+
+// New create service instance and return.
+func New(c *conf.Config) (s *Service) {
+	d := dao.New(c)
+	s = &Service{
+		c:   c,
+		dao: d,
+	}
+	go s.fetchData()
+	return
+}
+
+// Close close service.
+func (s *Service) Close() (err error) {
+	s.dao.Close()
+	return
+}
+
+// Ping ping service.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+func (s *Service) fetchData() {
+	for {
+		s.marketProc()
+		time.Sleep(time.Hour * 24)
+	}
+}