Create & Init Project...

app/job/openplatform/open-market/dao/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "berserker.go",
+        "dao.go",
+        "market_es.go",
+        "market_mysql.go",
+    ],
+    importpath = "go-common/app/job/openplatform/open-market/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/openplatform/open-market/conf:go_default_library",
+        "//app/job/openplatform/open-market/model:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/gopkg.in/olivere/elastic.v5:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/open-market/dao/berserker.go

@@ -0,0 +1,148 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+
+	"go-common/app/job/openplatform/open-market/model"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+
+	xhttp "net/http"
+
+	pkgerr "github.com/pkg/errors"
+)
+
+const (
+	_userAgent = "User-Agent"
+	_queryJSON = `{"select":[{"name":"item_id"},{"name":"pv"},{"name":"uv"},{"name":"days_before"}],"where":{"item_id":{"in":[%d]}},"page":{"limit":30,"skip":0}}`
+)
+
+var (
+	signParams = []string{"appKey", "timestamp", "version"}
+)
+
+// QueryPUVCount get puv info from berserker
+func (d *Dao) QueryPUVCount(c context.Context, itemID int32) (pv map[int32]int64, uv map[int32]int64, err error) {
+	pv = make(map[int32]int64)
+	uv = make(map[int32]int64)
+	v := make(url.Values, 8)
+	v.Set("appKey", d.c.Berserker.Appkey)
+	v.Set("signMethod", "md5")
+	v.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	v.Set("version", "1.0")
+	query := fmt.Sprintf(_queryJSON, itemID)
+	v.Set("query", query)
+	var res struct {
+		Code   int               `json:"code"`
+		Result []model.PUVResult `json:"result"`
+	}
+
+	if err = d.doHTTPRequest(c, d.c.Berserker.URL, "", v, &res); err != nil {
+		log.Error(d.c.Berserker.URL+"?"+v.Encode(), err)
+		return
+	}
+
+	if res.Code == 200 && len(res.Result) > 0 {
+		for _, v := range res.Result {
+			pv[v.DaysBefore] = v.PV
+			uv[v.DaysBefore] = v.UV
+		}
+		return
+	}
+	return
+}
+
+// doHttpRequest make a http request for data platform api
+func (d *Dao) doHTTPRequest(c context.Context, uri, ip string, params url.Values, res interface{}) (err error) {
+	enc, err := d.sign(params)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "uri:%s,params:%v", uri, params)
+		return
+	}
+	if enc != "" {
+		uri = uri + "?" + enc
+	}
+
+	req, err := xhttp.NewRequest(xhttp.MethodGet, uri, nil)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "method:%s,uri:%s", xhttp.MethodGet, uri)
+		return
+	}
+	req.Header.Set(_userAgent, "changxuanran@bilibili.com  "+env.AppID)
+	if err != nil {
+		return
+	}
+	return d.client.Do(c, req, res)
+}
+
+// Sign calc appkey and appsecret sign.
+func (d *Dao) sign(params url.Values) (query string, err error) {
+	tmp := params.Encode()
+	signTmp := d.encode(params)
+	if strings.IndexByte(tmp, '+') > -1 {
+		tmp = strings.Replace(tmp, "+", "%20", -1)
+	}
+	var b bytes.Buffer
+	b.WriteString(d.c.Berserker.Secret)
+	b.WriteString(signTmp)
+	b.WriteString(d.c.Berserker.Secret)
+	mh := md5.Sum(b.Bytes())
+	// query
+	var qb bytes.Buffer
+	qb.WriteString(tmp)
+	qb.WriteString("&sign=")
+	qb.WriteString(strings.ToUpper(hex.EncodeToString(mh[:])))
+	query = qb.String()
+	return
+}
+
+// Encode encodes the values into ``URL encoded'' form
+// ("bar=baz&foo=quux") sorted by key.
+func (d *Dao) encode(v url.Values) string {
+	if v == nil {
+		return ""
+	}
+	var buf bytes.Buffer
+	keys := make([]string, 0, len(v))
+	for k := range v {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		found := false
+		for _, p := range signParams {
+			if p == k {
+				found = true
+				break
+			}
+		}
+		if !found {
+			continue
+		}
+		vs := v[k]
+		prefix := k
+		for _, v := range vs {
+			buf.WriteString(prefix)
+			buf.WriteString(v)
+		}
+	}
+	return buf.String()
+}
+
+//func (d *Dao) reverse(s string) string {
+//	n := len(s)
+//	runes := make([]rune, n)
+//	for _, rune := range s {
+//		n--
+//		runes[n] = rune
+//	}
+//	return string(runes[n:])
+//}

app/job/openplatform/open-market/dao/dao.go

@@ -0,0 +1,88 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/openplatform/open-market/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+
+	elastic "gopkg.in/olivere/elastic.v5"
+)
+
+//Dao struct
+type Dao struct {
+	c *conf.Config
+	// http client
+	client *bm.Client
+	// db
+	ticketDB *sql.DB
+	//es
+	es    *elastic.Client
+	esUgc *elastic.Client
+}
+
+// New new a Dao and return.
+func New(c *conf.Config) (d *Dao) {
+	var (
+		err   error
+		es    *elastic.Client
+		esUgc *elastic.Client
+	)
+	d = &Dao{
+		c:        c,
+		client:   bm.NewClient(c.HTTPClient),
+		ticketDB: sql.NewMySQL(c.DB.TicketDB),
+	}
+	es, err = elastic.NewClient(
+		elastic.SetURL(c.ElasticSearch.Addr...),
+		elastic.SetSniff(false),
+		elastic.SetHealthcheckInterval(time.Duration(c.ElasticSearch.Check)),
+		elastic.SetErrorLog(&elog{}),
+		elastic.SetInfoLog(&ilog{}),
+	)
+	if err != nil {
+		panic(err)
+	}
+	esUgc, err = elastic.NewClient(
+		elastic.SetURL(c.ElasticSearchUgc.Addr...),
+		elastic.SetSniff(false),
+		elastic.SetHealthcheckInterval(time.Duration(c.ElasticSearch.Check)),
+		elastic.SetErrorLog(&elog{}),
+		elastic.SetInfoLog(&ilog{}),
+	)
+	if err != nil {
+		panic(err)
+	}
+	d.es = es
+	d.esUgc = esUgc
+	return d
+}
+
+// Ping ping health.
+func (d *Dao) Ping(c context.Context) (err error) {
+	return d.ticketDB.Ping(c)
+}
+
+// Close close.
+func (d *Dao) Close() (err error) {
+	if err = d.ticketDB.Close(); err != nil {
+		log.Error("dao.ticketDB.Close() error(%v)", err)
+	}
+	return
+}
+
+type ilog struct{}
+type elog struct{}
+
+// Printf printf.
+func (l *ilog) Printf(format string, v ...interface{}) {
+	log.Info(format, v...)
+}
+
+// Printf printf.
+func (l *elog) Printf(format string, v ...interface{}) {
+	log.Error(format, v...)
+}

app/job/openplatform/open-market/dao/market_es.go

@@ -0,0 +1,133 @@
+package dao
+
+import (
+	"context"
+	"gopkg.in/olivere/elastic.v5"
+	"strconv"
+	"time"
+
+	"go-common/app/job/openplatform/open-market/model"
+	"go-common/library/log"
+)
+
+var (
+	orderIndex   = "product_order"
+	orderType    = "product_order_info"
+	commentIndex = "open_mall"
+	commentType  = "ugc"
+	marketIndex  = "product_market"
+	marketType   = "product_market_info"
+)
+
+const (
+	_orderStatusComplete = 2
+	_dateFormat          = "2006-01-02"
+	_timeFormat          = "2006-01-02 15:04:05"
+)
+
+//OrderData fetch ordercount by project and days
+func (d *Dao) OrderData(c context.Context, projectID int32, startTimeUnix int64) (orderData map[int32]int64, err error) {
+	var (
+		searchResult *elastic.SearchResult
+		startTime    string
+		firstTime    string
+		firstDay     time.Time
+		startDay     time.Time
+		daysBefore   = -1
+	)
+	orderData = make(map[int32]int64)
+	startTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Format(_dateFormat)
+	firstTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Add(time.Hour * 24 * -30).Format(_dateFormat)
+	startDay, _ = time.Parse(_dateFormat, startTime)
+	firstDay, _ = time.Parse(_dateFormat, firstTime)
+	for {
+		daysBefore++
+		startDay = startDay.Add(time.Hour * -24)
+		if !(startDay.Before(firstDay)) {
+			rangeQuery := elastic.NewRangeQuery("pay_time")
+			rangeQuery.Gte(startDay.Format(_timeFormat))
+			rangeQuery.Lte(startDay.Add(time.Hour * 24).Format(_timeFormat))
+			query := elastic.NewBoolQuery()
+			query.Must(elastic.NewMatchQuery("project_id", projectID))
+			query.Must(elastic.NewMatchQuery("status", _orderStatusComplete))
+			query.Must(elastic.NewExistsQuery("pay_time"))
+			query.Must(rangeQuery)
+			searchResult, err = d.es.Search().
+				Index(orderIndex).Type(orderType).
+				Query(query).
+				Size(0).
+				Timeout(d.c.ElasticSearch.Timeout).
+				Do(c)
+			orderData[int32(daysBefore)] = searchResult.TotalHits()
+			continue
+		}
+		break
+	}
+	return
+}
+
+// CommentData get comment info from ugc es
+func (d *Dao) CommentData(c context.Context, projectID int32, startTimeUnix int64) (commentData map[int32]int64, err error) {
+	var (
+		searchResult *elastic.SearchResult
+		startTime    string
+		firstTime    string
+		firstDay     time.Time
+		startDay     time.Time
+		daysBefore   = -1
+	)
+	commentData = make(map[int32]int64)
+	startTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Format(_dateFormat)
+	firstTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Add(time.Hour * 24 * -30).Format(_dateFormat)
+	startDay, _ = time.Parse(_dateFormat, startTime)
+	firstDay, _ = time.Parse(_dateFormat, firstTime)
+	for {
+		daysBefore++
+		startDay = startDay.Add(time.Hour * -24)
+		if !(startDay.Before(firstDay)) {
+			rangeQuery := elastic.NewRangeQuery("ctime")
+			rangeQuery.Gte(startDay.Unix() * 1000)
+			rangeQuery.Lte(startDay.Add(time.Hour*24).Unix() * 1000)
+			query := elastic.NewBoolQuery()
+			query.Must(elastic.NewMatchQuery("subjectId", projectID))
+			query.Must(elastic.NewMatchQuery("subjectType", 2))
+			query.Must(rangeQuery)
+			searchResult, err = d.esUgc.Search().
+				Index(commentIndex).Type(commentType).
+				Query(query).
+				Size(0).
+				Timeout(d.c.ElasticSearch.Timeout).
+				Do(c)
+			commentData[int32(daysBefore)] = searchResult.TotalHits()
+			continue
+		}
+		break
+	}
+	return
+}
+
+//SaveData save result to es
+func (d *Dao) SaveData(c context.Context, project *model.Project) (err error) {
+	exists, err := d.es.IndexExists(marketIndex).Do(context.Background())
+	if err != nil {
+		log.Error("check if index exists error (%v)", err)
+		return
+	}
+	if !exists {
+		if _, err = d.es.CreateIndex(marketIndex).Do(c); err != nil {
+			log.Error("index name(%s) create err(%v)", marketIndex, err)
+			return
+		}
+	}
+	_, err = d.es.Index().
+		Index(marketIndex).
+		Type(marketType).
+		Id(strconv.Itoa(int(project.ID))).
+		BodyJson(project).
+		Refresh("true").
+		Do(c)
+	if err != nil {
+		log.Error("put es [%d] err(%v)", project.ID, err)
+	}
+	return
+}

app/job/openplatform/open-market/dao/market_mysql.go

@@ -0,0 +1,137 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/openplatform/open-market/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_fetchProjectList = "SELECT `id`,`name`,`start_time`,`type` FROM `project` WHERE `status` = 1 AND `type` IN (1,3,4,5,6,8)"
+
+	//_fetchUserWishFirst = "SELECT `mtime` FROM `user_wish` WHERE `item_id` = ? ORDER BY `mtime` LIMIT 1"
+	_fetchUserWishByDay = "SELECT DISTINCT `id` FROM `user_wish` WHERE `item_id` = ? AND `mtime` BETWEEN ? AND ?"
+
+	//_fetchUserFavoriteFirst = "SELECT `mtime` FROM `user_favorite` WHERE `item_id` = ? AND `status` = 1 AND `type` = 1 ORDER BY `mtime` LIMIT 1"
+	_fetchUserFavoriteByDay = "SELECT DISTINCT `id` FROM `user_favorite` WHERE `item_id` = ? AND `status` = 1 AND `type` = 1  AND `mtime` BETWEEN ? AND ? "
+
+	_fetchStockIfSoldOut = "SELECT `sku_id` FROM `sku_stock` WHERE `item_id` = ? AND `stock` <> 0  LIMIT 1"
+	//_fetchStockSoldOutDay = "SELECT `mtime` FROM `sku_stock` WHERE `item_id` = ? AND `stock` = 0 ORDER BY `mtime` DESC LIMIT 1"
+)
+
+//FetchProject query project list from mysql
+func (d *Dao) FetchProject(c context.Context) (projectList []*model.Project, err error) {
+	var rows *sql.Rows
+	if rows, err = d.ticketDB.Query(c, _fetchProjectList); err != nil {
+		log.Error("d._fetchProjectListSQL.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		project := new(model.Project)
+		if err = rows.Scan(&project.ID, &project.Name, &project.StartTime, &project.Type); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			projectList = nil
+			return
+		}
+		//if project.ID == 11644 || project.ID == 11678 || project.ID == 11643 || project.ID == 11650 {
+		//	continue
+		//}
+		if soldOut := d.CheckStock(context.TODO(), project.ID); soldOut {
+			continue
+		}
+		projectList = append(projectList, project)
+	}
+	err = rows.Err()
+	return
+}
+
+//CheckStock if a project sold out,return date
+func (d *Dao) CheckStock(c context.Context, projectID int32) (soldOut bool) {
+	row := d.ticketDB.QueryRow(c, _fetchStockIfSoldOut, projectID)
+	var skuID int
+	if err := row.Scan(&skuID); err != nil {
+		if err == sql.ErrNoRows {
+			return true
+		}
+		log.Error("row.Scan() error(%v)", err)
+	}
+	return
+}
+
+//WishData fetch wishcount by project and date
+func (d *Dao) WishData(c context.Context, projectID int32, startTimeUnix int64) (wishData map[int32]int64, err error) {
+	var (
+		startTime  string
+		firstTime  string
+		firstDay   time.Time
+		startDay   time.Time
+		daysBefore = -1
+	)
+	wishData = make(map[int32]int64)
+	startTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Format(_dateFormat)
+	firstTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Add(time.Hour * 24 * -30).Format(_dateFormat)
+	startDay, _ = time.Parse(_dateFormat, startTime)
+	firstDay, _ = time.Parse(_dateFormat, firstTime)
+	for {
+		daysBefore++
+		startDay = startDay.Add(time.Hour * -24)
+		if !(startDay.Before(firstDay)) {
+			var rows *sql.Rows
+			count := 0
+			if rows, err = d.ticketDB.Query(c, _fetchUserWishByDay, projectID, startDay.Format(_dateFormat), startDay.Add(time.Hour*24).Format(_dateFormat)); err != nil {
+				log.Error("wish count query error(%v)", err)
+				return
+			}
+			for rows.Next() {
+				count++
+			}
+			err = rows.Err()
+			wishData[int32(daysBefore)] = int64(count)
+			rows.Close()
+			continue
+		}
+		break
+	}
+	return
+}
+
+//FavoriteData fetch favoritecount by project and date
+func (d *Dao) FavoriteData(c context.Context, projectID int32, startTimeUnix int64) (favoriteData map[int32]int64, err error) {
+	var (
+		startTime  string
+		firstTime  string
+		firstDay   time.Time
+		startDay   time.Time
+		daysBefore = -1
+	)
+	favoriteData = make(map[int32]int64)
+	startTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Format(_dateFormat)
+	firstTime = time.Unix(startTimeUnix, 0).Add(time.Hour * 24).Add(time.Hour * 24 * -30).Format(_dateFormat)
+	startDay, _ = time.Parse(_dateFormat, startTime)
+	firstDay, _ = time.Parse(_dateFormat, firstTime)
+	for {
+		daysBefore++
+		startDay = startDay.Add(time.Hour * -24)
+		if !(startDay.Before(firstDay)) {
+			var rows *sql.Rows
+			count := 0
+			if rows, err = d.ticketDB.Query(c, _fetchUserFavoriteByDay, projectID, startDay.Format(_dateFormat), startDay.Add(time.Hour*24).Format(_dateFormat)); err != nil {
+				log.Error("wish count query error(%v)", err)
+				return
+			}
+			for rows.Next() {
+				count++
+			}
+			err = rows.Err()
+			favoriteData[int32(daysBefore)] = int64(count)
+			rows.Close()
+			continue
+		}
+		break
+	}
+	return
+}